
    io@              $       &   d dl mZ d dlZd dlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ ddgZ G d de      Zd	e d
e	 de de de de_        dee   dee   dee   dee   dee   dee   dedededededededededdf dZdee   dee   dee   dee   dee   dee   dedededededededededdf dZ ee      	 	 	 	 	 d"dee   dee   dee   dee   dee   dee   d edz  dedededededededededdf"d!       Zy)#    )castN)Tensor   )_capturable_doc_default_to_fused_or_foreach_differentiable_doc_disable_dynamo_if_unsupported_foreach_doc!_get_capturable_supported_devices_get_scalar_dtype
_get_value_maximize_doc_params_doc
_to_scalar_use_grad_for_differentiable_view_as_real	OptimizerParamsTASGDasgdc                        e Zd Z	 	 	 	 	 	 	 	 	 ddedeez  dedededededz  d	ed
ededdf fdZ fdZd Z	e
dd       Z xZS )r   Nparamslrlambdalphat0weight_decayforeachmaximizedifferentiable
capturablereturnc           
          t        |t              r|j                         dk7  rt        d      d|k  st        d|       d|k  st        d|       ||||||||	|
d	}t        |   ||       y )Nr   zTensor lr must be 1-elementg        zInvalid learning rate: zInvalid weight_decay value: )	r   r   r   r   r   r   r   r    r!   )
isinstancer   numel
ValueErrorsuper__init__)selfr   r   r   r   r   r   r   r   r    r!   defaults	__class__s               J/var/www/html/engine/venv/lib/python3.12/site-packages/torch/optim/asgd.pyr(   zASGD.__init__   s     b&!bhhjAo:;;by6rd;<<l";L>JKK ( ,$

 	*    c                    t         |   |       | j                  D ]f  }|j                  dd        |j                  dd       |j                  dd       |j                  dd       |d   D ]  }| j                  j                  |g       }t        |      dk7  s/t        j                  |d         s;t        |d         }t        j                  |t               |j                  	      |d<   t        j                  |d
         s0t        j                  |d
   t               |j                  	      |d
<   t        j                  |d         rt        j                  |d   t               |j                  	      |d<    i y )Nr   r   Fr    r!   r   r   step)dtypedeviceetamu)r'   __setstate__param_groups
setdefaultstategetlentorch	is_tensorfloattensorr   r1   )r)   r7   grouppp_statestep_valr+   s         r,   r4   zASGD.__setstate__?   sF   U#&& 	EY-Z/-u5\518_ **..B/w<1$ ??76?;#(#9*/,,$,=,?+ !??75>:).#EN2C2Eahh* !??74=9(-#DM1B1DQXX)	r-   c                    d}|d   D ]  }	|	j                   |t        j                  |	      z  }|j                  |	       |	j                   j                  rt        d      |j                  |	j                          | j                  |	   }
t        |
      dk(  rt        j                  d|	j                  t                     |
d<   t        j                  t        |d         |	j                  t                     j                         j                         |
d	<   t        j                  d|	j                  t                     |
d
<   t        j                   |	t        j"                        |
d<   |j                  |
d
          |j                  |
d          |j                  |
d	          |j                  |
d           |S )NFr   z&ASGD does not support sparse gradientsr    )r1   r0   r/   r   r2   r3   )memory_formatax)gradr:   
is_complexappend	is_sparseRuntimeErrorr7   r9   zerosr1   r   	as_tensorr   clonedetachones
zeros_likepreserve_format)r)   r>   params_with_gradgradsmusaxsetasstate_stepshas_complexr?   r7   s              r,   _init_groupzASGD._init_groupW   s{   x !	2Avv!u//22 ''*66##&'OPPQVV$

1u:?$)KK1883D3F%E&M &uT{3#$88"3"5
  %L #(**1883D3F#E$K #("2"2)>)>#E$K 

5;'

5;'E%L)""5=1C!	2D r-   c                 b   | j                          d}|$t        j                         5   |       }ddd       | j                  D ][  }g }g }g }g }g }g }	| j	                  |||||||	      }
t        ||||||	|d   |d   |d   |d   |d   |d   |d   |d	   |d
   |
       ] |S # 1 sw Y   uxY w)zPerform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r    r!   )
r   r   r   r   r   r   r   r    r!   rX   ) _cuda_graph_capture_health_checkr:   enable_gradr5   rY   r   )r)   closurelossr>   rR   rS   rT   rU   rV   rW   rX   s              r,   r/   z	ASGD.step}   s    	--/""$ !y! && 	E-/"$E "C "C!#D(*K**'S$K  Gn;;Gn">2i(z*$%56 .'!	> E! !s   B%%B.)	g{Gz?g-C6?g      ?g    .Ar   NFFFN)__name__
__module____qualname__r   r<   r   boolr(   r4   rY   r   r/   __classcell__)r+   s   @r,   r   r      s     "#$ ++ FN+ 	+
 + + + + + + + 
+B0$L "- "-r-   zImplements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Args:
        am  
        lr (float, Tensor, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        z

    .. _Acceleration of stochastic approximation by averaging:
        https://meyn.ece.ufl.edu/wp-content/uploads/sites/77/archive/spm_files/Courses/ECE555-2011/555media/poljud92.pdf

    r   rS   rU   rT   rV   rW   r   r   r   r   r   r   r    r!   rX   r"   c       	   
      0   t         j                  j                         st        |      }t	        |       D ]  \  }}||   }|s|n| }||   }||   }||   }||   }t         j
                  j                         s|rt               }|j                  j                  |j                  j                  cxk(  r3|j                  j                  cxk(  r|j                  j                  k(  rn n|j                  j                  |v st        d| d      t        j                  |      r?t        j                  |      }t        j                  |      }t        j                  |      }|dz  }|
dk7  r|j                  ||
      }|r,|j                  d||z  z
         |j                  ||d       n6t!        |      }|j                  d||z  z
         |j#                  ||        |s|j%                         dk7  r0|j#                  |j'                  |      j                  |             n|j)                  |       |r`|j)                  |d||z  |z  z   |	z  z         |j)                  dt        j*                  ||z
  t        j,                  |            z         it!        |      }t        j.                  |d||z  |z  z   |	z  z        }|j)                  |       t        j.                  dt1        d||z
        z        }|j)                  |        y )NUIf capturable=True, params, mus, etas, and state_steps must be on supported devices: .r   r   r   value)r:   jitis_scriptingr   	enumeratecompileris_compilingr   r1   typeAssertionErrorrG   view_as_realaddmul_addcmul_r   add_itemsubcopy_maximum	ones_likerL   max)r   rS   rU   rT   rV   rW   r   r   r   r   r   r   r    r!   rX   iparamrF   r3   rE   r2   step_tcapturable_supported_devices	eta_valuer/   new_etanew_mus                              r,   _single_tensor_asgdr      s   $ 99!!#^f% 95Qx#t$VV1gQ ~~**,+L+N(!!99>>&::??& ==%%& LL%%)EE$--I,J!M 
 E"%%d+D&&u-E##B'B 	!188E86DJJq53;'NN4BN/"3IJJq59,,-JJtI:J. aGGEIIbM&&r*+HHUOIIbQf!44>?@HHQv{EOOF4KLLMf%DoobQd1B-Bu,L&MNGIIg__QQr	):%:;FHHVs9r-   c       	         p  " t        |       dk(  ry |rt        d      t        j                  j	                         s?|r=t        d      "t        "fdt        | |||d      D              st        d" d	      t        |      }t        j                  | |||||g      }|j                         D ]x  \  \  }}\  \  }}}}}}}t        t        t           |      }t        t        t           |      }t        t        t           |      }t        t        t           |      }t        t        t           |      }t        t        t           |      }|rt        |||       |rt        j                   |      }t        j                  j	                         s=|d   j"                  r.t        j$                  |t        j&                  d
d      d
       nt        j$                  |d       |
dk7  rN|rt        j$                  |||
       |}nt        j(                  |||
      }t        j$                  |||       nt        j(                  |||      }t        j*                  |||d       ~t        j,                  ||      }t        j*                  |||       ~|rt        j,                  ||      }t        j.                  |d
       t        j0                  |       t        j2                  ||       ~t        j4                  ||      } t        j6                  | |       t        j$                  | d       t        j8                  | |	       t        j0                  |        t        j6                  | |       t        j2                  ||        |D !cg c](  }!t        j:                  |d||z  |!z  z   |	z  z  |      * } }!|D !cg c]2  }!t        j:                  dt=        dt?        |!      |z
        z  |      4 }}!t        j2                  ||        t        j2                  ||       { y c c}!w c c}!w )Nr   z#_foreach ops don't support autogradF)supports_xlac              3   ,  K   | ]  \  }}}}|j                   j                  |j                   j                  cxk(  xr5 |j                   j                  cxk(  xr |j                   j                  k(  nc xr |j                   j                  v   y wr_   )r1   rq   ).0r?   r3   r2   r/   r   s        r,   	<genexpr>z%_multi_tensor_asgd.<locals>.<genexpr>3  so      
 !2sD HHMMRYY^^RszzR$++BRBRR >!==>
s   BBT)strictrf   rg   g      ?cpu)r1   rh   r   ri   rj   ) r9   rr   r:   ro   rp   r   allzipr   r   "_group_tensors_by_device_and_dtypeitemsr   listr   r   _foreach_negis_cpu_foreach_add_r=   _foreach_add_foreach_addcmul__foreach_sub_foreach_maximum__foreach_reciprocal__foreach_copy__foreach_mul_foreach_mul__foreach_pow_rL   r}   r   )#r   rS   rU   rT   rV   rW   r   r   r   r   r   r   r    r!   rX   grouped_tensorsr1   _grouped_params_grouped_grads_grouped_axs_grouped_mus_grouped_etas_grouped_state_steps_grouped_paramsgrouped_gradsgrouped_axsgrouped_musgrouped_etasgrouped_state_stepsintermediatenew_musnew_etasr/   r   s#                                     @r,   _multi_tensor_asgdr     s   $ 6{aBCC >>&&(Z'H(
$  
 %(T;t$T
 

 !&&B%C1F 
 
BBBB	S$4O 
			 c7 
	 
	
 	d6lO<T&\>:4<64<6DL-8"4<1EF.-E!..}=M ~~**,1DQ1G1N1N#U\\#e%DC  3Q7 1##M>V,$11!>  nEJ --~UL 	lRTU )).+F\;G (()<bAG##GS1&&w/  g6 ))*=uEH"-!,%0&&x0"-  x8 0 q52:+<'<&F GPVWH  0 C:d+;b+@$A A&QG    x8  g6Gc7ts   -P.7P3)single_tensor_fnr   c                   |t        | |d      \  }}|r)t        j                  j                         rt	        d      |r%t        j                  j                         st
        }nt        } || |||||||||||||	|
       y)znFunctional API that performs asgd algorithm computation.

    See :class:`~torch.optim.ASGD` for details.
    NF)	use_fusedz6torch.jit.script not supported with foreach optimizers)	r   r   r   r   r   r   r    r!   rX   )r   r:   rl   rm   rJ   r   r   )r   rS   rU   rT   rV   rW   r   r   r    r!   rX   r   r   r   r   r   r   funcs                     r,   r   r     s    4 1Ne

7 599))+STTuyy--/!"!%r-   )NFFFF)typingr   r:   r   	optimizerr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   __all__r   __doc__r   r<   rc   r   r   r   rC   r-   r,   <module>r      s         & 6
N9 Nb	 
 	 
 		 		 		 .NLN<N 
fN 
f	N
 v,N fN N 	N 	N N N N N N  !N" 
#NbO7LO7<O7 
fO7 
f	O7
 v,O7 fO7 O7 	O7 	O7 O7 O7 O7 O7 O7  !O7" 
#O7d  1DE   6L6<6 
f6 
f	6
 v,6 f6 D[6 6 6 6 6 6  	!6" 	#6$ %6& '6( 
)6 F6r-   