iM#JUdZddlZddlmZddlmZmZddlmZddl Z ddl m Z edZ edZ iZee j j"efed <ehd Zd ed ed ee e fdeee e fgee e fffdZ d9ddddde j0de j0de j0dee j0dededede j0fdZedde d9ddddde j0de j0de j0dee j0dededede j0fdZdeededefdZd e j0d!edede j0fd"Zd#e j0d$e j0d%ed&edeed'ede j0fd(Zd%ed&eddfd)Z d#e j0d$e j0d%ed&edeede j0f d*Z! d:dddddd+dd,d#e j0d$e j0d-e j0d.ee j0d/ee j0d0ee j0d1ed2ed3ed'edeed4ed5eede"e j0e j0e j0e j0ffd6Z#ed7de# d:dddddd+dd,d#e j0d$e j0d-e j0d.ee j0d/ee j0d0ee j0d1ed2ed3ed'edeed4ed5eede"e j0e j0e j0e j0ffd8Z$y);zImplementations of ONNX operators as native Torch ops. NOTE: Fake implementations: Refer to https://docs.pytorch.org/docs/stable/library.html#torch.library.register_fake for more details on how to create fake kernels. N)Callable)OptionalTypeVar) ParamSpec)_dtype_mappings_P_RONNX_ATEN_DECOMP_TABLE> op_type opset_version fake_implreturnchdtttfdtttfffd }|S)zDDecorator to register an ONNX operator with a custom implementation.funcrcd}tjjdd|d|}|tt t tj j |<|j|S)Nopsetzonnx::.) mutates_args)torchlibrary custom_opr getattropsonnx register_fake)roverloadtorch_oprrrs N/var/www/html/engine/venv/lib/python3.12/site-packages/torch/onnx/ops/_impl.py decoratorz_onnx_op..decorator's|=/*==**WIQxj )+   wwuyy~~w'GRS y))rrr )rrrr$s``` r#_onnx_opr&"s0 R( Xb"f-=  r%F) interleaved num_headsrotary_embedding_dimx cos_cache sin_cache position_idsr'r(r)c"|jS)zFFake implementation for RotaryEmbedding-23 for torch.compile purposes.)clone)r*r+r,r-r'r(r)s r#_rotary_embedding_23_fake_implr05s 779r%RotaryEmbeddingcV|jt}ddtjj dk(fdtjjdk(fdtjjdk(fdtjj dk(xrj dk(fd n@tjj d k(xrj d k(fd |d k(rtj |d }nG|d k(rBtj|dk7fdd}||z} || g} tj || }tjt|jd k(d|jd } |dk(r| }|ddddddd|f} |dddddd|df} |dz ntjjdk(xrjdk(fdtjjdk(xrjdk(fdtjjdk(fdtjjdk(fdtjdtjd|r%| dddddddddf} | dddddddddf}ntj| dd\} }| z|zz }| z|zz}|rftj|d}tj|d}tj||fd}tj || j} ntj||fd} tj| | fd}|d k(rtj |Stj |d S)z_RotaryEmbedding-23 https://onnx.ai/onnx/operators/onnx__RotaryEmbedding.html#rotaryembedding-23rNc"djS)Nz6position_ids must be 2D when provided. Received shape shape)r-sr#z%rotary_embedding_23..ZsL\M_M_L`ar%c.ddjdS)Nz6position_ids first dim (batch) must match x.shape[0] ( ). Received rr7) batch_sizer-sr#r9z%rotary_embedding_23..^s$LZLXdeqewewxyezd{|r%r c.ddjdS)Nz;position_ids second dim (sequence) must match x.shape[-2] (r;r r7)r-sequence_lengthsr#r9z%rotary_embedding_23..bs=QRaQbbno{pBpBCDpEoFGr%c<djdjS)NzWcos_cache/sin_cache must be 2D when position_ids is provided. Received cos_cache shape , sin_cache shape r7r+r,sr#r9z%rotary_embedding_23..f&((1'88J9??J[]r%c<djdjS)Nz[cos_cache/sin_cache must be 3D when position_ids is not provided. Received cos_cache shape r@r7rAsr#r9z%rotary_embedding_23..lrBr%)rr5r rCcdS)NzKnum_heads must be provided for 3D inputs. Received input tensor with shape r) input_shapesr#r9z%rotary_embedding_23..ysabmanor%cy)Nzx should be a 4D tensor by nowrrr%r#r9z%rotary_embedding_23..r%c0djdddS)Nzcos has shape  but expected (batch=, seq=, ...)r7)r<cosr>sr#r9z%rotary_embedding_23..$. +@ FSbRccijr%c0djdddS)Nzsin has shape rKrLrMr7)r<r>sinsr#r9z%rotary_embedding_23..rOr%c0djdddS)NzLast dimension of cos cache (rR') should match rotary_embedding_dim/2 ().r7)rNrotary_embedding_dim_halfsr#r9z%rotary_embedding_23.../ " >effAACDr%c0djdddS)NzLast dimension of sin cache (rRrTrUr7)rVrQsr#r9z%rotary_embedding_23..rWr%dim) r8lenr_checkrZpermutereshape unsqueezechunkcat)r*r+r,r-r'r(r) input_rank hidden_size head_size new_shapex_rotate x_not_rotatex1x2realimagx_rotate_concatoutputr<rNrGrVr>rQs ``` @@@@@@r#rotary_embedding_23rnCs''K[!JQJ!"oO    ! # a     q !Z / |     q !_ 4 G   MMOq 9Y]]_%9 ]  MMOq 9Y]]_%9 ] Q MM!\ * q N o "!n 9, )YG MM!Y ' LLQWW"$LM Iq (Aq////0HQ12334L 4 9     LL ! "Fsyy|'Fj LL ! "Fsyy|'Fj LL " 22 D LL " 22 D // Q C // Q C  aAqt!tm $ aAqt!tm $Xqb1B 8cBh D 8cBh DtR(tR())T4Lb9==(..A99dD\r2 YY,/R 8FQ}}V[11 == ..r%scalerdc:||Sdtj|z S)z/Get the scale factor for attention computation.g?)mathsqrt)rords r#_get_scale_factorrss %5GC$))I2F,FGr%tensorr<c|jd|jd}}||z}|j||||jddjS)z1Reshape 3D tensor to 4D for multi-head attention.r r5)r8view transpose contiguous)rtr<r(r>rcrds r#_reshape_3d_to_4drysP$*<<?FLLO[Oy(I JIF 1a r%QKcurrent_q_num_headscurrent_kv_num_headsqk_matmul_output_modec |dk(rt|||||Stjtj||j ddS)z1Get QK output tensor based on the specified mode.rr4rR)_compute_qk_output_for_mode_0r zeros_likematmulrw)rzr{r|r}ror~s r#_get_qk_output_for_aten_spdarsO!, q%';U   Q B0C DEEr%cHtjzdk(fdy)z-Validate Group Query Attention configuration.rcdddS)Nz q_num_heads (z%) must be divisible by kv_num_heads (z ) for GQAr)r}r|sr#r9z-_validate_gqa_configuration..s- 344YZnYooxyr%N)rr\)r|r}s``r#_validate_gqa_configurationrs" LL22a7yr%c|}||k7r||z}|j|d}t||jd}tj|}||z} ||z} t j | | jddS)zDHelper function to compute QK output for qk_matmul_output_mode == 0.r rYrCr4rR)repeat_interleaversr8rqrrrrrw) rzr{r|r}roK_for_qk repeat_factor scale_factor sqrt_scaleQ_scaledK_scaleds r#rrsH22+/CC &&}!&<$UAGGAJ7L<(J:~H*$H <<("4"4R"< ==r%) is_causal kv_num_heads q_num_headsr~rosoftcapsoftmax_precisionV attn_maskpast_key past_valuerrrrrc|jd} t|jdk(r|jd}|j}|4| ||jd|jdz|jd|zf}n#| ||jd|jd|zf}|}| |||df}n|jd}|j}|K|jd|jd|jd|jdz|jdf}n |j}|}|jd|jd|jd|df}tj||j|j }tj||j|j }tj||j|j }tj||j|j }||||fS)z@Fake implementation for Attention-23 for torch.compile purposes.rrCr r5dtypedevice)r8r[remptyrr)rzr{rrrrrrrr~rorrr<q_sequence_length output_shapepresent_key_shapepresent_value_shapeqk_output_shaperm present_key present_value qk_outputs r#_attention_23_fake_implrs"J 177|qGGAJww   q!AGGAJ. l* !   l* !  0    a  GGAJww     q!AGGAJ. ! !" / GGAJ GGAJ GGAJ a  [[QWWQXX FF++/qwwqxxPKKK 3177188TM O177188LI ; y 88r% Attentionc  d\} }}t|j}|jd}t|jdk(rWtj|dk7xr|dk7d|jd}t |||}t |||}t |||}tjt|jdk(xr2t|jdk(xrt|jdk(d|j|}t | |} |tj ||g| n|j}|tj ||g| n|j}||}}|j| }|j| }|j|}|j|}| d k(xr0| dk(xr)| duxr#|duxs|jtjk(}t|||rOtjjj||||d || t||k7 }t||||| | }nY||k7r+||z}|j|| }|j|| }tj |||j|j" }|rytj|dud tj$tj&||tj|j" }|j)|t+d}|?|jtjk(r|j)|t+d}n||z}t | |jd}t-j.|} || z}!|| z}"tj0|!|"j3dd}#|#}|#|z}$| dk(r|$}| d kDr| tj4|$| z z}$| dk(r|$}| w| t6vrW|$j}%|$j9t:j<| }$tj>|$d }&|&j9|%}&n/tj>|$d }&ntj>|$d }&| dk(r|&}tj0|&|}|dk(r1|j3ddjAjC||d}||||fS)zMAttention-23 https://onnx.ai/onnx/operators/onnx__Attention.html#attention-23)r r5rCrrCcy)Nz;q_num_heads and kv_num_heads must be provided for 3D inputsrrr%r#r9zattention_23..rIr%r rEcy)Nz'Q, K, and V should be 4D tensors by nowrrr%r#r9zattention_23..rIr%NrYr)r dropout_prro enable_gqarcy)Nz'Cannot use both is_causal and attn_maskrrr%r#r9zattention_23..rIr%z-infr4rRr5)"r[r8rr\ryrsrar/rboolrnn functionalscaled_dot_product_attentionrrzerosrtrilones masked_fillfloatrqrrrrwtanh-_ATTENTION_23_ALLOWED_INTERMEDIATE_PRECISIONStorONNX_DTYPE_TO_TORCH_DTYPEsoftmaxrxrv)'rzr{rrrrrrrr~rorr num_head_dim sequence_dimhead_diminput_shape_lenr<r q_head_sizerrr|r}kv_sequence_length can_use_sdparmrr attn_bias causal_maskrrrrqk_matmul_output qk_with_biasoriginal_dtype qk_softmaxs' r# attention_23rcs&,3(L,!''lOJ 177|q 1  2!2 Q GGAJ a[ 9 a\ : a\ : LL AGG Ec!''la/ECLA4E9 ''(#K e[ 1E    8Q-\2 WWY  !  :q/|4 WWY  qA'',/77<0 -. 3 A !Q & A  % A$  ?)//UZZ"?  35IJ$$AA #';;B  1   !   "6 6/3GGM##M|#DA##M|#DAKK 1  LLT!#T  ** %&**88 K"--{lE&MJI  %**,%119*eFmL & 1 ) ; YY|, z>z>!<<(2D2DR2LM% ()3 A %$I S="UZZ w0F%GGL A %$I  ( $QQ!-!3!3+#==>OP  #]]: "]]rs, $$' *t_ T]ACUZZ22H<=B091- !$19"b&1A xB (2r6"223.,0   !  || || || 5<<(      \\  R!?@ ,0 D/  !D/ ||D/||D/||D/5<<( D/  D/D/D/ \\D/AD/NHXe_HHH  LL &) 69  \\ F ||F ||FF F E? F  F \\F$47 > ||> ||>> > E? >  \\ >4)-'+)- Q9!"!'+Q9 ||Q9 ||Q9 ||Q9 % Q9 u||$ Q9 & Q9Q9Q9Q9Q9 E?Q9Q9 }Q9 5<<u||U\\ ABQ9h +r23 )-'+)- 9!"!'+9 ||9 ||9 ||9 % 9 u||$ 9 & 99999 E?99 }9 5<<u||U\\ AB949r%