iC UddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z m Z ddlmZmZddlmZmZddlmZdd lmZmZdd lmZdd lmZdd lmZmZmZm Z ejBe"Z#ejHe%d <e dGddZ&de'e(dee&fdZ)de'e(dee&fdZ*d,de(de(de+e'e(ee&ffdZ,de(fdZ-dddddd d!d"d#Z.Gd$d%e Z/Gd&d'e Z0Gd(d)Z1d*e2ddfd+Z3y)-N)Iterable)ThreadPoolExecutor) dataclass)BaseHTTPRequestHandlerThreadingHTTPServer)parse_qsurlparse) DictLoader Environment)tabulate)get_world_sizetcpstore_client)build_db) JobConfig) CollectiveGroup MembershipNCCLCallloggerT)slotsc.eZdZUeed<eed<dZdZy)Response status_codetextcl|jdk7r%td|jd|jy)NzHTTP z: )r RuntimeErrorrselfs [/var/www/html/engine/venv/lib/python3.12/site-packages/torch/distributed/debug/_frontend.pyraise_for_statuszResponse.raise_for_status"s8   s "t'7'7&8499+FG G #c@tj|jSN)jsonloadsrrs r r%z Response.json&szz$))$$r"N)__name__ __module__ __qualname__int__annotations__strr!r%r"r rrs IH%r"rurlsreturncddld}dtdtffd }t|5}|j ||}ddd|S#1swYSxYw)Nrurlr/cfj|}t|j|jSr$)postrrr)r2resprequestss r getzfetch_thread_pool..get0s(}}S!(($))44r") max_workers)r6r,rrmap)r.r8r7executorrespsr6s @r fetch_thread_poolr<*sXK555  4( S$'( L( Ls AA cddldjdtdtfddttdt tffd }t j||S)Nrsessionr2r/cK|j|4d{}|jd{}t|j|cdddd{S7C7-7 #1d{7swYyxYwwr$)r4rrstatus)r>r2r5rs r fetchzfetch_aiohttp..fetch>sc<<$ / /$DDKK. / / /$ / / / /sTA7AA7A"AA" A7A A7A" A7"A4(A+ )A40A7r.c Kj4d{}tj|Dcgc] }|| c}d{cdddd{S7Dcc}w77#1d{7swYyxYwwr$) ClientSessionasynciogather)r.r>r2aiohttprAs r rEzfetch_aiohttp..gatherCst((* P Pg )N#%*=)NOO P P P)NO P P P PsaA=AA=A(A A(A$A( A=A&A=A(&A=(A:.A1 /A:6A=)rFrCr,rlistrrDrun)r.rErFrAs @@r fetch_aiohttprI:sZ/W22/// P49P();P ;;vd| $$r"endpointargsc6t}ttDcgc]}d| }}|j|}|Dcgc]}|j d|d|}} t |}||fScc}wcc}w#t $rt|}Y||fSwxYw)Nrankz /handler/?)rranger multi_getdecoderI ImportErrorr<)rJrKstorerkeysaddrsaddrr;s r fetch_allrXJs  E %n&6 7 81d1#J 8D 8 OOD !EFK Ld iz4& 9 LE L)e$ %< 9 L )!%( %<)s A4A9% A>>BBblobcZtj|}tj|dS)N)indent)r%r&dumps)rYparseds r format_jsonr_Xs! ZZ F ::fQ ''r"a {% block title %}{% endblock %} - PyTorch Distributed
{% block header %}{% endblock %} {% block content %}{% endblock %}
z {% extends "base.html" %} {% block header %}

{% block title %}Index{% endblock %}

{% endblock %} {% block content %} Hi {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}{{title}}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ format_json(resp.text) }}
{% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}torch.profiler{% endblock %}

{% endblock %} {% block content %}
{% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} {% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}TCPStore Keys{% endblock %}

{% endblock %} {% block content %} 
    {% for k, v in zip(keys, values) -%}
{{ k }}: {{ v | truncate(100) }}
    {% endfor %}
{% endblock %} ag {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %}

Groups

{{ groups | safe }}

Memberships

{{ memberships | safe }}

Collectives

{{ collectives | safe }}

NCCL Calls

{{ ncclcalls | safe }} {% endblock %} aQ {% extends "base.html" %} {% block header %}

{% block title %}py-spy Stack Traces{% endblock %}

{% endblock %} {% block content %}
{% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} )z base.html index.html raw_resp.htmljson_resp.html profile.html tcpstore.html fr_trace.htmlpyspy_dump.htmlcReZdZUejZej ed<dZe ed<y)_IPv6HTTPServeraddress_familyirequest_queue_sizeN) r'r(r)socketAF_INET6ri AddressFamilyr+rjr*r-r"r rhrhJs!+1??NF((:""r"rhc veZdZUded<dZdZdefdZdeee effdZ defdZ d efd ed e d e de fd Zy )HTTPRequestHandlerFrontendServerfrontendcRtjd|jd||zy)Nz%s %sr)rinfoclient_address)rformatrKs r log_messagezHTTPRequestHandler.log_messageRs&     " TM r"c:|jj|yr$)rq_handle_requestrs r do_GETzHTTPRequestHandler.do_GETYs %%d+r"r/c@t|jjSr$)r pathrs r get_pathzHTTPRequestHandler.get_path\s "'''r"c4t|jSr$)r get_raw_queryrs r get_queryzHTTPRequestHandler.get_query_s**,--r"c@t|jjSr$)r r{queryrs r r~z HTTPRequestHandler.get_raw_querybs "(((r"NnamedefaulttypecJ|j}||vr|S|||dS)Nr)r)rrrrrs r get_query_argz HTTPRequestHandler.get_query_arges0  u NE$KN##r")r'r(r)r+rvryr,r|dictrGrr~objectrrr-r"r roroOso ,(#(.4T#Y/.)s),0c$$"($7;$ $r"roc eZdZdefdZddZddZdeddfdZd e d e de fd Z dede fd Z dede fd Zdede fdZdee deede fdZdede fdZdede fdZdede fdZdede fdZdede fdZdede fdZdede fdZy)rpportc ptt}t|d|_|jjj t tt|j|j|j|j|j|j|j|j |j"|j$d |_t)dt*fd|i}d|f}t-|||_t1j2|j4ddd |_|j6j9y) NT)loader enable_async)zipr_ enumerate) /z/stacksz /pyspy_dumpz /fr_tracez/fr_trace_jsonz/fr_trace_ncclz/fr_trace_nccl_jsonz/profilez/wait_countersz /tcpstorerorqr-z distributed.debug.FrontendServer)targetrKdaemonr)r templatesr _jinja_envglobalsupdaterr_r _handle_index_handle_stacks_handle_pyspy_dump_handle_fr_trace_handle_fr_trace_json_handle_fr_trace_nccl_handle_fr_trace_nccl_json_handle_profiler_handle_wait_counters_handle_tcpstore_routesrrorh_server threadingThread_serve_threadstart)rrrRequestHandlerClassserver_addresss r __init__zFrontendServer.__init__osI&%V$G &&# ' ##**22.."88"88#'#B#B--"88..   #  !    d&~7JK  '';;3   r"r/Nc |jjy#t$rtj dYywxYw)Nz got exception in frontend server)r serve_forever Exceptionr exceptionrs r rzFrontendServer._serves8 A LL & & ( A   ? @ As >>c8|jjyr$)rjoinrs r rzFrontendServer.joins r"reqc|j}||jvr|jdd|y|j|} ||}|jd|jdd|j|jj|y#ttf$r>}t j d||jddt|Yd}~yd}~wwxYw) NizHandler not found: z-Exception in frontend server when handling %siz Exception: rz Content-typez text/html) r|r send_errorr SystemExitrrrepr send_response send_header end_headerswfilewrite)rrr{handlerr5es r rxzFrontendServer._handle_requests||~ t|| # NN3"5dV < = ,,t$ 3 OOGLL!  &F KK4fnn6P6PQ KKMr")r)4rDr%rrkrcollections.abcrconcurrent.futuresr dataclassesr http.serverrr urllib.parserr jinja2r r r torch.distributed.debug._storer r4torch.distributed.flight_recorder.components.builderr;torch.distributed.flight_recorder.components.config_managerr2torch.distributed.flight_recorder.components.typesrrrr getLoggerr'rLoggerr+rrGr,r<rItuplerXr_rrhrorpr*rr-r"r rsY  $1!C+*JIQ+**844  % % % DI (8*<  %S %hx&8 %  3 d3i(AS6S0T (c( GP""?@  cj  Z#)# $/$>sPsPlstr"