o j9:j9@sBUddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddl m Z ddlmZddlmZmZddlmZmZddlmZmZdd lmZmZeeZeje d <d Z!e"e d <ed dGdddZ#ed dGdddZ$ed dGdddZ%GdddeZ&de'e(de"de'e#fddZ)de'e(de"de'e#fddZ* d=de(d e(de"de+e'e(e'e#ffd!d"Z,d#e(fd$d%Z-d&e'e(d'e'e#de(dBfd(d)Z.d*Z/d+Z0d,Z1Gd-d.d.Z2Gd/d0d0eZ3Gd1d2d2eZ4Gd3d4d4Z5 d>d5e6d6e(dBd7e"d8e'e&d9e7e(d:e"ddfd;d<Z8dS)?N)ABCabstractmethod)Callable)ThreadPoolExecutor) dataclass)BaseHTTPRequestHandlerThreadingHTTPServer)parse_qsurlparse) DictLoader Environment)get_world_sizetcpstore_clientloggerN@_DEFAULT_FETCH_TIMEOUTT)slotsc@s.eZdZUeed<eed<ddZddZdS)Response status_codetextcCs&|jdkrtd|jd|jdS)NzHTTP : )r RuntimeErrorrselfrh/home/nk/hobo-godmode/plappi-mvp/.venv/lib/python3.10/site-packages/torch/distributed/debug/_frontend.pyraise_for_status#s zResponse.raise_for_statuscCs t|jSN)jsonloadsrrrrrr' z Response.jsonN)__name__ __module__ __qualname__int__annotations__strrrrrrrrs  rc@seZdZUeed<eed<dS)NavLinkpathlabelN)r"r#r$r'r&rrrrr(+s  r(c@s(eZdZUeed<edgefed<dS)Router)HTTPRequestHandlerhandlerN)r"r#r$r'r&rbytesrrrrr+1s r+c@s|eZdZUeZeed<edee fddZ edee fddZ de eeffddZded Bfd d Zdefd d Zd S) DebugHandler fetch_timeoutreturncCdSrrrrrrroutes:zDebugHandler.routescCr2rrrrrr nav_links=r4zDebugHandler.nav_linkscCsiSrrrrrr templates@r4zDebugHandler.templatesNcCr2rrrrrrdumpCr4zDebugHandler.dumpcCst|jSr)typer"lowerrrrr dump_filenameFzDebugHandler.dump_filename)r"r#r$rr0floatr&rlistr+r3r(r5dictr'r6r7r:rrrrr/7s r/urlstimeoutr1csdddld}dtdtffdd }t|d}t|||}Wd|S1s+wY|S)Nrurlr1c szj|d}t|j|jWSjjy+}ztdd|WYd}~Sd}~wjjyF}ztdd|WYd}~Sd}~wtyd}ztdt|j d|WYd}~Sd}~ww)Nr@z Timeout: zConnectionError: r) postrrr exceptionsTimeoutConnectionError Exceptionr8r")rBresperequestsr@rrgetUs&zfetch_thread_pool..get) max_workers)rOr'rrr=map)r?r@rQrPexecutorrespsrrNrfetch_thread_poolOs  rUcsTddldjdtdtffdd dttdttffdd }t||S) NrsessionrBr1c sz0||4IdH}|IdH}t|j|WdIdHWS1IdHs*wYWdStjyL}ztdd|WYd}~Sd}~wjyk}ztdt|jd|WYd}~Sd}~wt y}ztdt|jd|WYd}~Sd}~ww)NrDzTimeoutError: rErrF) rGrrstatusasyncio TimeoutError ClientErrorr8r"rK)rVrBrLrrM)aiohttprrfetchks  4&&zfetch_aiohttp..fetchr?c svjd}j|d4IdHttjfdd|DIdHWdIdHS1IdHs4wYdS)N)totalrCcsg|]}|qSrr).0rB)r\rVrr zz1fetch_aiohttp..gather..) ClientTimeout ClientSessionr=rXgather)r?client_timeoutr[r\r@)rVrrcws  "0zfetch_aiohttp..gather)r[rbr'rr=rXrun)r?r@rcrrer fetch_aiohttpfs" rgendpointargscstt}ddttD}||}fdd|D}z t||d}W||fSty9t||d}Y||fSw)NcSsg|]}d|qS)rankr)r^rrrrr_r`zfetch_all..cs$g|]}|ddqS)z /handler/?)decode)r^addrrjrirrr_s$rC)rranger multi_getrg ImportErrorrU)rirjr@storekeysaddrsrTrrpr fetch_alls  rwblobcCst|}tj|ddS)N)indent)rr dumps)rxparsedrrr format_jsons r}rvrTcCsrddt|D}|s dSt|}|t|}d|d|dg}|D]\}}|d|d|jq#d |S) zHReturn a summary string if any workers failed, or None if all succeeded.cSs"g|] \}}|jdkr||fqS)r)r)r^irlrrrr_s"z(format_fetch_summary..NzPARTIAL DATA: /z workers respondedz Rank r ) enumeratelenappendrjoin)rvrTfailedr]oklinesrkrLrrrformat_fetch_summarys   rac {% block title %}{% endblock %} - PyTorch Distributed
{% block header %}{% endblock %} {% block content %}{% endblock %}
a {% extends "base.html" %} {% block header %}

{% block title %}{{title}}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ format_json(resp.text) }}
{% endif %} {% endfor %} {% endblock %} c @sLeZdZ ddeedededdfddZdd d Zdd d Z dd dZ dS)PeriodicDumperrhandlers output_dirinterval_secondsr1NcCs&||_||_||_t|_d|_dSr) _handlers _output_dir_interval_seconds threadingEvent _stop_event_thread)rrrrrrr__init__s   zPeriodicDumper.__init__cCs2tj|jddtj|jddd|_|jdS)NT)exist_okz distributed.debug.PeriodicDumper)targetdaemonname)osmakedirsrrThread_runrstartrrrrrszPeriodicDumper.startcCs&|j|jdur|jdSdSr)rsetrrrrrrstop&s  zPeriodicDumper.stopc Cs|jsx|jD]a}z|}Wnty!td|Yqw|dur'qt d}|d|d}t j |j |}zt|d }||Wdn1sTwYWqtyitd|Yqw|j|j|jrdSdS)NzFailed to dump %sz %Y%m%d_%H%M%S_z.txtwzFailed to write dump to %s)ris_setrr7rKr exceptionr:timestrftimerr)rropenwritewaitr)rr-content timestampfilenamer)frrrr+s.        zPeriodicDumper._runrr1N) r"r#r$r=r/r'r<rrrrrrrrrs rc@s*eZdZUejZejed<dZe ed<dS)_IPv6HTTPServeraddress_familyirequest_queue_sizeN) r"r#r$socketAF_INET6r AddressFamilyr&rr%rrrrrEs rc @s|eZdZUded<ddZddZdefdd Zdeee effd d Z defd d Z defdede de de fddZdS)r,FrontendServerfrontendcGstd|jd||dS)Nz%s %sr)rinfoclient_address)rformatrjrrr log_messageMs zHTTPRequestHandler.log_messagecCs|j|dSr)r_handle_requestrrrrdo_GETTszHTTPRequestHandler.do_GETr1cCs t|jjSr)r r)rrrrget_pathWr!zHTTPRequestHandler.get_pathcCs t|Sr)r get_raw_queryrrrr get_queryZr!zHTTPRequestHandler.get_querycCs t|jjSr)r r)queryrrrrr]r!z HTTPRequestHandler.get_raw_queryNrdefaultr8cCs$|}||vr |S|||dS)Nr)r)rrrr8rrrr get_query_arg`sz HTTPRequestHandler.get_query_arg)r"r#r$r&rrr'rr>r=rrobjectr8rrrrrr,Js" r,c@sfeZdZ ddedeedBfddZdddZdd d Zd e ddfd d Z de de de fddZdS)rNportrc Cs|dur ddlm}|}ddd|D}tttd}|D] }||qt|}t |dd|_ |j j jt t t|d i|_|D]}|D] }|j|j|j<qJqD||_td tfd |i} d |f} t| | |_tj|jd ddd|_|jdS)Nr)default_handlersrcss4|]}|D]}d|jd|jdVqqdS)z z N)r5r)r*)r^r-linkrrr us z*FrontendServer.__init__..)z base.htmlz raw_resp.htmlzjson_resp.htmlT)loader enable_async)zipr}rr5r,rrhrz distributed.debug.FrontendServer)rrjrr)'torch.distributed.debug._debug_handlersrr BASE_TEMPLATERAW_RESP_TEMPLATEJSON_RESP_TEMPLATEupdater6r r _jinja_envglobalsrr}r_routesr3r-r)rr8r,r_serverrr_serverr) rrrrnav_html all_templatesr-rrouteRequestHandlerClassserver_addressrrrrjsP    zFrontendServer.__init__r1cCs0z|jWdStytdYdSw)Nz got exception in frontend server)r serve_foreverrKrrrrrrrs  zFrontendServer._servecCs|jdSr)rrrrrrrr;zFrontendServer.joinreqc Cs|}||jvr|dd|dS|j|}z||}Wn&ttfyE}ztd||ddt|WYd}~dSd}~ww|d| dd| |j |dS) NizHandler not found: z-Exception in frontend server when handling %siz Exception: rz Content-typez text/html) rr send_errorrK SystemExitrrrepr send_response send_header end_headerswfiler)rrr)r-rLrMrrrrs(     zFrontendServer._handle_requesttemplatekwargscKs|j|jdi|S)Nr)r get_templaterenderencode)rrrrrrrender_templateszFrontendServer.render_templaterr)r"r#r$r%r=r/rrrr,rr'rr.rrrrrris  = rrdump_dir dump_intervalr enabled_dumpsr0c s|D]}||_qttjt||d}td|jjd}|dur:t fdd|D||}| td||z| W|durJ| dSdS|durU| ww)N)rrz"Frontend server started on port %dcsg|] }|vr|qSr)r:)r^r-rrrr_s  zmain..z2Periodic dumper started, writing to %s every %.0fs) r0rsetLevelloggingINFOrrr server_portrrrr) rrrrrr0r-serverdumperrrrmains6       r)rhr)9rXrrrrrrabcrrcollections.abcrconcurrent.futuresr dataclassesr http.serverrr urllib.parser r jinja2r r torch.distributed.debug._storer r getLoggerr"rLoggerr&rr<rr(r+r/r=r'rUrgtuplerwr}rrrrrrr,rr%rrrrrrsz       "@5h