
    iIE                     |   d Z ddlZddlZddlmZ ddlmZmZmZ ddl	Z	ddl
mZ ddlmZ ddlmZ ddlmZ d	d
lmZmZ d	dlmZmZmZmZmZmZ d	dlmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) erddl*m+Z+  G d de,      Z- G d de,      Z. ee/d      Z0de	jb                  de2fdZ3 G d de)      Z4y)a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
    N)Iterable)AnyOptionalTYPE_CHECKING)TensorVariable)Source)getArtifactLogger)tree_map_only   )GuardBuilderinstall_guard)
AttrSourceConstDictKeySourceDictGetItemSourceGetItemSourceGlobalWeakRefSource
GradSource)GLOBAL_KEY_PREFIX   )VariableTracker)ConstantVariable)ConstDictVariable)ListVariable)GetAttrVariable)UserDefinedObjectVariable)InstructionTranslatorc                       e Zd Zy)ArgMappingExceptionN__name__
__module____qualname__     [/var/www/html/engine/venv/lib/python3.12/site-packages/torch/_dynamo/variables/optimizer.pyr   r   8       r$   r   c                       e Zd Zy)GuardInstallExceptionNr   r#   r$   r%   r(   r(   <   r&   r$   r(   
perf_hintsxreturnc                    ddl m} | j                  ry || j                  j                  d      }t
        j                  j                  j                  |       d u}|r-|j                  J |xs |j                  j                  |       S |S y)Nr   )get_managerFT)torch._inductor.cudagraph_treesr-   is_cudadeviceindextorch_dynamoutilsget_static_address_typecurrent_node_is_cuda_graph_recorded_tensor)r*   r-   manageris_static_addresss       r%   _is_static_for_cudagraphsr:   C   s    ;yyahhnne4!MM//GGJRVV''333! J''FFqI
 %$ r$   c                       e Zd Zdddhej                  Z	 	 	 ddej                  j                  dee	e
ef      deee      dee	ej                  ef      de
ddf fdZd	d
dedee   de	eef   ddf
 fdZd	d
dedef fdZddZddZde
de
deee
   e	ee
f   f   fdZddZddZd	d
dej                  defdZd	d
dee   de
dee
   de
ddfdZddZ xZ S )OptimizerVariablegrad_to_sourcetensor_to_sourcestatic_tensor_namesNvaluekwargsr+   c                     t        |   |fi | || _        |xs i | _        |xs i | _        |xs
 t               | _        y N)super__init__r@   r=   r>   setr?   )selfr@   r=   r?   r>   rA   	__class__s         r%   rE   zOptimizerVariable.__init__^   sG     	)&),1
,2 0 6B#6#?#% r$   txr   nameargsr   c                 `   |dk(  rt        | j                  d      st        
|   ||||      S 	 | j	                  |       | j                           | j                  |i |\  }} | j                  j                  |i |}| j                  |       | j                  |||||       dt        | j                         }|j                  || j                         | j                  |       t        j                  |      S t        
|   ||||      S # t        t         f$ r
}	Y d}	~	&d}	~	ww xY w)zVThis is an optimization to avoid tracing the very slow initialization of the optimizer_init_group__optimizer_N)hasattrr@   rD   call_methodgraph_break_if_pending_mutationmove_step_if_cpuget_python_argsrM   map_sources_and_install_guardsupdate_list_argsidstore_global_weakref_by_idcreate_finalizerr   creater   r(   )rG   rI   rJ   rK   rA   py_args	py_kwargsret_valmangled_name_rH   s             r%   rP   zOptimizerVariable.call_methodm   s*    = 4::}5w*2tT6BB44R8%%'%9T%9%94%J6%J"0$**00'GYG33B7%%b$K ".bn-=>--lDJJG%%b) (..w77
 w"2tT6::	 ()>? s   CD D-(D-c                 "   |dv r0| j                   sJ t        | |t        | j                   |            S |dk(  rFddlm} | j
                  j                  D ]  }|d   D ]  } ||d         | j                  |       t        | %  ||      S )	NrM   )sourceparam_groupsr   mark_static_addressparamsTguard)
r`   r   r   
decoratorsrc   r@   ra   _set_capturablerD   var_getattr)rG   rI   rJ   rc   groupprH   s         r%   ri   zOptimizerVariable.var_getattr   s     M";;;"4jd6STT>!800 7x 7A'677   $w"2t,,r$   c           	         | j                   j                  D ]t  }|d   D ]j  }|j                  j                  }|j                  j                  t        |      d       }|sA|j                  |      sSddlm	}  |dd| d| dg        l v y )	Nrd   r   )unimplementedz(optimizer: pending mutation on parameterz
variable: z, parameter: zSPending mutations on a parameter (e.g. due to using closure) require a graph break.)gb_typecontextexplanationhints)
r@   ra   outputside_effectsid_to_variablegetrV   has_pending_mutationexcrm   )rG   rI   grk   rs   variablerm   s          r%   rQ   z1OptimizerVariable.graph_break_if_pending_mutation   s    
 (( 	Ax[ !yy55'66::2a5$G A A( K3! J",XJmA3 G$y 		r$   c                     ddl m} dt        t        t        f   dt
        f fd} j                  j                  D ]  } ||      sd|d<     j                  xr t         j                  d      }|j                  t        j                  | j                  j                  |            }|j                  D ]L  }t        j                  t!        j"                  d            }t!        j"                  d      |j                  |<   N y )	Nr   LazyVariableTrackerrj   r+   c                     d}d}| j                  dg       D ]:  }||j                  xs |j                  z  }||j                  j                  vz  }< d| v xr |xr |S )NTrd   
capturable)ru   r/   is_xpur@   state)rj   all_uninitializedall_gpurk   rG   s       r%   safe_to_set_capturablezAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturable   sq     $GYYx, ?19900!Qdjj.>.>%>>!?  5(J->J7Jr$   Tr~   ra   ) r|   dictstrr   boolr@   ra   r`   r   realize_allr   builditemsr   _HashableTrackerr   rY   )	rG   rI   r|   r   rj   r`   param_groups_vtparam_group_vtkeys	   `        r%   rh   z!OptimizerVariable._set_capturable   s    )	K$sCx. 	KT 	K ZZ,, 	+E%e,&*l#	+ HDKK!H-99!!"djj&=&=vF
 .33 	FN#44 ''5C )9(?(?(EN  %		Fr$   c                      dt         dt         f fd}|D cg c]
  } ||       }}|j                         D ci c]  \  }}| ||       }}}||fS c c}w c c}}w )z9Get python values equivalent to the variable tracker argsargr+   c                    t        | t              r | j                         r| j                         S t        | t              r| j
                  sg S t        | t              rt        | j                  t              rtt        | j                  j                  t              rP| j                  j                  j                  dk(  r-j                  j                  | j                  j                     S t        )Nra   )
isinstancer   is_python_constantas_python_constantr   r   r   r`   r   baser   memberr@   ra   r1   r   )r   rG   s    r%   map_argz2OptimizerVariable.get_python_args.<locals>.map_arg   s    #/C4J4J4L--//C.syy	3 12szz=9szz
;JJOO**n<zz..szz/?/?@@%%r$   )r   r   )	rG   rK   rA   r   r   new_argskv
new_kwargss	   `        r%   rS   z!OptimizerVariable.get_python_args   sg    
	& 	& 	& -11SGCL1106?1am?
?## 2?s
   AAc                     | j                   j                  j                         D ];  \  }}d|v s|d   j                  s|d   j	                  |j
                        |d<   = y )Nstep)r@   r   r   is_cputor0   )rG   rk   r   s      r%   rR   z"OptimizerVariable.move_step_if_cpu   sW    

((..0 	;HAu5=#7#7 %f 0 0 :f	;r$   c                    ddl m ddlm} i | _        i | _        dt        dd ffd}t        t        j                  || j                  j                         | j                  xr t        | j                  d      }|j                  t        j                   || j                  j"                  |            }| j                  xr t        | j                  d	      }t        j                   || j                  j                  |      }|j%                          |J |j&                  j(                  j+                  |       t-        | j                  j"                  |j.                        D ]  \  }}	t1        |d
         dkD  r|d
   D ]  }
|
j2                  d }t5        | j                  j                  j7                               D ]  \  }}||
u s|} n |sR|j                  t        j                   || j                  j                  |
   t9        |t;        ||                          n |	j=                  |t?        j@                  d
            }d}g }t-        |d
   |jC                  |            D ]  \  }}|j                  }|| j
                  |<   tE        |d      }|j2                  C|| j                  |j2                  <   tG        |j2                        rhd}|jI                  |       |tK        |jM                  tN        jP                                |rtR        jU                  tV        jX                        s|D cg c]  }|jZ                   }}tR        j]                  d|       
 t5        | j                  j                  j_                               D ]  \  }}t9        |t;        ||            }|j&                  j(                  j+                  |       t5        |j_                               D ]a  \  }}ta        |t        j                        s!|| j                  vs0|| j
                  vs?t9        |t;        ||            | j
                  |<   c  y c c}w )Nr   rb   r   r{   r*   r+   c                      | d       y )NTre   r#   )r*   rc   s    r%   mark_staticzEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_static  s    .r$   ra   r   rd   r   TgradF)zGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.)1rg   rc   lazyr|   r=   r>   r   r
   r2   Tensorr@   r   r`   r   r   r   r   ra   realizerr   guard_on_key_orderaddzipr   lenr   	enumeratekeysr   r   getitem_constr   rY   unpack_var_sequencer   r:   appendr   
make_guardr   CONSTANT_MATCHperf_hint_logisEnabledForloggingDEBUGrJ   warningvaluesr   )rG   rI   r|   r   params_groups_sourcer   state_sourcestate_vtrj   group_vtparam	key_indexir   	params_vt
all_staticnon_static_gradsrk   p_vtparam_sourcegrad_sourcesrcnon_static_grad_namesidxr@   p_state_source	inner_idxr   rc   s                               @r%   rT   z0OptimizerVariable.map_sources_and_install_guards   s   4-  "	/3 	/4 	/ 	ellK1A1AB  ${{Vz$++~/V-99!!"djj&=&=?ST
 {{Gz$++w'G"((TZZ-=-=|L 	'''
		$$((6  #4::#:#:O<Q<QR 6	OE8 5?#a'"8_ "Ezz-$(	$-djj.>.>.C.C.E$F &DAq Ez,-	 %& %/;; / 5 5$&$(JJ$4$4U$;$5(4(:<(S%&!"	 "%"( !..r3C3J3J83TUIJ!uX	0M0Mb0QR W4#{{+7%%a(( 
 66%2=D''/4QVV<%*
(//<!+"8"89T9T"UVW$ -"<"<W]]"K=M(Nc(N%(N%%
 *_6	t $DJJ$4$4$;$;$=> 	JC.0sCN II((,,^< )%,,. 9 	1q%,,/!4!44!6!66/@&(:>9(U0D))!,	 )Os   Q"tensor_valuec                    ddl m} || j                  v rX ||d       | j                  |   }| j                  j	                  |j
                  j                  |j                               n|| j                  v r| j                  |   }ni ||d       |j                  t        |      }t        |      }| j                  j	                  |j
                  j                  |j                               t        j                  |||      S )z%Wrap state tensor in a TensorVariabler   rb   Tre   )rg   rc   r>   r?   r   rr   module_key_namerJ   r=   rW   r   r   r   r   )rG   rI   r   rc   r`   global_names         r%   wrap_tensorzOptimizerVariable.wrap_tensora  s     	5 4000D9**<8F$$(()B)B6;;)OPT000((6F  D9778I<XK(5F$$(()B)B6;;)OP$$Rv>>r$   rZ   r[   c           	      $   t        ||      D ]   \  }}t        |t              st        |t              sJ d       t	        |      D ]  \  }}	|j
                  j                  j                  |       t        |	t        j                        r,|j                  j                  | j                  ||	             q|j                  xr t        |j                  |      }
|j                  j                  t        j                   ||	|
               y)z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)r   r   r   listr   rr   rs   mutationr2   r   r   r   r   r`   r   r   r   )rG   rI   rK   rA   rZ   r[   r   py_argr   valr`   s              r%   rU   z"OptimizerVariable.update_list_args}  s     tW- 	QKC#|,!&$/ C/ (/ QFAsII**33C8!#u||4		(()9)9"c)BC!$!Lcjj!0L		(()>)>r3)OPQ	Qr$   c                     | j                   | j                  |j                  j                  dt        j
                  j                  dd ffd}|j                  j                  |       y )Ngmr+   c                 D     d fd}t        j                  |       y )Nc                     D ]  } j                   j                  | d        j                  j                  | d        j                  rj                  j	                          j
                  snj
                  j	                           y rC   )_bufferspop_parametersparams_flatclearparams_flat_unwrap_subclasses)rJ   r   names_to_deletetcs    r%   clear_static_tensor_refsz\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refs  sl    + ADKKOOD$/NN&&tT2~~,,.7788>>@Ar$   r+   N)weakreffinalize)r   r   r   r   r@   s   ` r%   init_finalizerz:OptimizerVariable.create_finalizer.<locals>.init_finalizer  s    A U$<=r$   )r?   r@   rr   tracing_contextr2   fxGraphModuleadd_graph_finalizer)rG   rI   r   r   r   r@   s      @@@r%   rX   z"OptimizerVariable.create_finalizer  sW    22

YY&&
	>uxx33 
	> 
	> 			%%n5r$   )NNN)rI   r   r+   Nr   )!r    r!   r"   r   _nonvar_fieldsr2   optim	Optimizerr   r   r   r   rF   r   r   r   rE   r   r   rP   ri   rQ   rh   tuplerS   rR   rT   r   r   r   rU   rX   __classcell__)rH   s   @r%   r<   r<   V   s    
#	1	1	N ;?26AE@{{$$@ !c:o!67@ &c#h/	@
 #4f(<#=>@ @ 
@";#"; "; ?#	";
 S/)*"; 
";H-5 -S -_ -&&F@$$$'$	tCy$sCx.(	)$<;
eN?)?9>?	?8Q#Q 'Q 	Q
 #Q Q 
Q,6r$   r<   )5__doc__r   r   collections.abcr   typingr   r   r   r2   torch._dynamo.variables.tensorr   torch._guardsr   torch._loggingr	   torch.utils._pytreer
   guardsr   r   r`   r   r   r   r   r   r   r4   r   r   r   constantr   dictsr   listsr   miscr   user_definedr   torch._dynamo.symbolic_convertr   	Exceptionr   r(   r    r   r   r   r:   r<   r#   r$   r%   <module>r      s   ,   $ / /  9   , - 0  & ! & $  ! 3 D	) 		I 	 "(L9 $ &N61 N6r$   