
    i             !       t   U d dl Z d dlZd dlZd dlZd dlZd dlmZmZ d dlm	Z	 d dl
mZ d dl mZmZ d dlmZmZ d dlmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmc mZ  d dlm!Z!m"Z"m#Z# d d	l$m%Z% d d
l&m'Z' d dlm(Z(m)Z)m*Z*m+Z+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z4 d dl5m6Z6 ejn                  jp                  Z8g Z9e:e;   e<d<   ejz                  j|                  j~                  Z? G d de      Z@	 	 ddedej                  deBdeBfdZC eeCej                  j                  d      ZE eeCej                  j                        ZF eeCej                  j                  d      ZG eeCej                  j                        ZIde#deJde#fdZK e%e?j                         e1d       eFd!e#d"e#fd#                     ZL e%e?j                         e1d       eFd!e#d"e#fd$                     ZM e%e?j                         e1d       eFd!e#de#d%eOd&eOfd'                     ZN e%e?j                         e1d       eFd(e#d)eOd*eOd+eOd,eBd-e#fd.                     ZP e%e?j                  j                  g      d/        ZS e%e?j                  jF                  g      d0e#fd1       ZT e%e?j                         e1       eFd2e#de#fd3                     ZU e%e?j                         e1d       eFd(e#d2e#fd4                     ZV e%e?j                         e1d       d(e#d2e#d5eOd6eOfd7              ZW e%e?j                         e1       eFd2e#de#fd8                     ZX e%e?j                         e1       eFd(e#d2e#de#fd9                     ZY e%e?j                         e1d       d(e#d2e#d&eOfd:              ZZ e%e?j                         e1d       eFd(e#d2e#d;eOd<eBfd=                     Z[ e%e?j                         e1d       eFdd>e#d2e#d?e;fd@                     Z\ e%e?j                        eFd(e#dAe#fdB              Z] e%e?j                         e1       eFd2e#de#fdC                     Z^ e%e?j                         e1d       eFd(e#d2e#de#fdD                     Z_ e%e?j                        d2e#dEe#de#fdF       Z` e%e?j                        d(e#d2e#dEe#debe#e#f   fdG       Za e%e?j                         e1       eFd(e#d2e#dHe#dIeOdJeOdKeBd<eBde#fdL                     Zc e%e?j                         e1d       eFd(e#d2e#dMe#de#fdN                     ZddOe#dPeJfdQZedRej                  fdSZg e%e?j                         e1       eFe@j                  j                  fd2e#dTe#dPeJde#fdU                     Zh e%e?j                         e1d       eFd(e#dAe#dTe#dPeJfdV                     Zk e%e?j                        ddW       Zm e%e?j                         e1       eFe@j                  j                  dXfd2e#dTe#dPeJd%eOfdY                     Zn e%e?j                  j                        eFd(e#d2e#dTe#dPeJd%eOf
dZ              Zo e%e?j                  j                        eFd(e#d2e#dTe#dPeJd%eOd e#fd[              Zr e%e?j                  j                        eFd(e#d2e#dTe#dPeJd\eOf
d]              Zs e%e?j                  j                        eFd(e#d2e#dTe#dPeJd\eOd e#fd^              Zud(e#d2e#dTe#dEee#   dPeJd_eJd`e#de#fdaZv e%e?j                         e1d       eFd(e#d2e#deJde#fdb                     Zw e%e?j                         e1d       d(e#d2e#dTe#dEee#   dPeJd_eJd`e#de#fdc              Zx e%e?j                         e1d       d(e#d2e#dTe#dEee#   dPeJd_eJd`e#de#fdd              Zy e%e?j                         e1       eFde@j                  j                  fd2e#dTe#dEee#   dPeJde#f
de                     Zz e%e?j                         e1d       eFde@j                  j                  fd(e#d2e#dTe#dEee#   dPeJde#fdf                     Z{ e%e?j                         e1       eFe@j                  j                  fdAe#dTe#dPeJde#fdg                     Z| e%e?j                         e1d       eFe@j                  j                  fd(e#d2e#dTe#dPeJde#f
dh                     Z} e%e?j                         e1       ddAe#die#djeOfdk              Z~ e%e?j                         e1       dle#dme#de#fdn              Z e%e?j                          e1       d(e#doe:eJ   deJdpeJdqeJdreJfds              Z e%e?j                  jF                        	 	 	 	 dd2e#deJdpeeJ   dqeeJ   dreJf
du       Zde#deJdpeeJ   dqeeJ   debeJeJf   f
dvZ e%e?j                         e1       	 	 	 	 ddAe#dwe#deJdpeeJ   dqeeJ   dreJfdx              Z e%e?j
                         e1       d(e#doe:eJ   deJdyeJfdz              Z e%e?j                         e1       d(e#doe:eJ   d{eJd|eJd}eJf
d~              Zd(e#d e#dej                  fdZ e%e?j                         e1d       eEd(e#de#deJdej                  fd                     Z e%e?j                         e1       eEd(e#de#deJdej                  fd                     Zd Z e%e?j                         e1       dAe#de:eJ   de:eJ   de:eJ   de:eJ   de#fd              Z e%e?j                         e1       eFdAe#de:eJ   de:eJ   de:eJ   de:eJ   de:eJ   de#fd                     Z e%e?j                         e1       d(e#de#d*eOfd              Z e%e?j                         e1       d>e#de:eJ   deJdeJdreJde#fd              Z e%e?j                  j                        eF	 dd(e#d2e#deeO   de#fd              Z e%e?j                         e?j                   j                  j#                  e8j$                        e?j                   j                  j#                  e8j&                        dAe#djeOdeeB   fd                     Z e%e?j(                         e1dd      dAe#djeOdeeB   fd              Z e%e?j*                         e1       de#deJdeBfd              Z e%e?j,                         e1d      de#deJdeBfd              Z e%e?j.                         e1       	 	 	 ddEe#de#deJdeBdeBde#fd              Z e%e?j0                         e1       d(e#de#deJdeJdeBf
d              Zde:eJ   fdZde:e#   deJdeJde:e#   fdZde:e#   fdZde:e#   deJfdZde:e#   deJdeJfdZ e%e?j<                  j                  e?j<                  j                  g      	 dde:e#   deJdeJdee#   de#f
d       Z e%e?j>                  j                  e?j>                  j                  g      	 	 dd2e#de:eJ   deJdee:e#      dee:e#      f
d       Z e%e?j@                  jF                        ddAe#deJdeJdebe#df   fd       Z e%e?jB                  j                        	 ddAe#de:eJ   deJdebe#df   fd       Z e%e?jD                  jF                        dd2e#deJdeJdebe#df   fd       Ze?jF                  jH                  j#                  e8j$                        	 dd2e#de#deJdebe#df   fd       Z e%e?jL                         e1d      eFdd2e#de#de#d%eJd)eJf
d                     Z e%e?jN                         e1       eF	 	 	 dd2e#de#de#d%eJd)eJdeBfd                     Z e%e?jP                         e1d      eFdd2e#de#de#d%eJd)eJf
d                     Z e%e?jR                  j                        eFd(e#dAe#de#de#dee#   deJdeJdeJdeJde:eB   debee#   ee#   ee#   f   fdń              Z e%e?jR                  j                        d(e#dAe#de#de#dee#   deJdeJdeJdeJde:eB   dejF                  dejF                  dejF                  debee#   ee#   ee#   f   fdǄ       Zdee#   dee#   fdȄZ e%e?jX                  j                        de#dAe#de:eJ   de#de#dEee#   dee#   de:eB   debee#   ee#   ee#   f   fd̄       Z e%e?jX                  j                        de#dAe#de:eJ   de#de#dEee#   dee#   de:eB   dejF                  dejF                  dejF                  debee#   ee#   ee#   f   fd̈́       Z e%e?j\                  j                        dAe#de:eJ   dEee#   deeO   debe#e#f   f
d΄       Z e%e?j^                  j                        de#dAe#de:eJ   de#dEee#   de:eB   debee#   ee#   f   fdτ       ZdAe#dEee#   dee#   dee#   dee#   dKeBdeOdeOdeBdebe#e#e#ee#   ee#   f   fdԄZ e%e?jb                         e1ddd֫      dAe#dEee#   dee#   dee#   dee#   dKeBdeOdeOdebe#e#e#f   fdׄ              Ze?jb                  j                  j#                  e8j&                        e?jb                  j                  j#                  e8j$                        dAe#dEee#   dee#   dee#   dee#   dKeBdeOdeOdebe#e#e#f   fd؄              Ze?jf                  j                  j#                  e8j$                        dde:e#   fdل       Z e%e?jj                  j                        dAe#dEee#   dee#   de#de#deOdeOdebe#e#e#f   fdڄ       Z e%e?jl                  j                        dAe#dEee#   dee#   de#de#dKeBdeOdeOdebe#e#e#f   fdۄ       Z e%e?jl                  jn                        dAe#dEee#   dee#   dKeBdeOdeOdebe#e#e#f   fd܄       Z e%e?jr                  j                        dAe#dEee#   dee#   de#de#dKeBdeOdeOdebe#e#e#e#e#f   fd݄       ZdAe#dEee#   dee#   de#de#deOdKeBde#fdބZ e%e?jv                  j                        dAe#dEee#   dee#   de#de#deOdeOdebe#e#e#e#f   fd߄       Z e%e?jx                  j                        dAe#dEee#   dee#   de#de#deOdeOdebe#e#e#e#e#e#f   fd       Z e%e?jz                  j                        dAe#dEee#   dee#   de#de#deOdeOdebe#e#e#e#f   fd       Z e%e?j|                         e1dd      eFdd                     Z e%e?j                         e1       ddddddddee#e)f   dReej                     deej                     deBdeBdeej                     fd              Z e%e?j                  e?j                  e?j                  g       e1       d               Ze?j                  j                  j#                  e8j&                         e%e?j                         e1dddd      dAe#dEe#dee#   dee#   dee#   dKeBdeOdeOfd                     Zd Z e%e?j                  j                        de#dAe#dEee#   dee#   dee#   dee#   dee#   deBdeOde:eB   de#debe#ee#   ee#   f   fd       Z e%e?j                  j                        de#dAe#dEee#   dee#   dee#   dee#   dee#   deBdeOde:eB   debe#ee#   ee#   f   fd       Z e%e?j                  j                        de#dAe#dEee#   dee#   dee#   dee#   dee#   deBdeOde:eB   dejF                  dejF                  dejF                  debe#ee#   ee#   f   fd       Z e%e?j                         e1dddƫ      dAe#d(e#dEe#dee#   dee#   dee#   dee#   deOfd              Z e%e?j                         e1dddƫ      dAe#d(e#dEe#dee#   dee#   dee#   dee#   deOde#fd              Z e%e?j                         e1       eFdAe#debeJeJf   fd                     Zd2e+de+de:eJ   deJfdZ e%e?j                         e1       d2e+de+de:eJ   fd              Z e%e?j                         e1       dAe+de+de:eJ   de:eJ   de:eJ   f
d              Z e%e?j                        dtdde+deJdye+de+d)e)f
d       Z e%e?j                         e1       dtdde+deJdye+de+d)e)f
d              Zdtdde+deJdye+de+deBd)e)fd Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        dd              Z e%e?j                        de+deJdye+de+fd       Z e%e?j                         e1       de+deJdye+de+fd              Zde+deJdye+de+deBf
dZ e%e?j                         e1ddM      eFd2e#debe#e#f   fd                     Z e%e?j                         e1       	 	 	 dde#deeBeJeOf   deeBeJeOf   deej                     fd	              Z e%e?j                        dd
       Zݐd Zސd Z e%e?j                  j                         e%e?j                  j                         e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        dAe#dee:eJ      dee:eO      de#fd                                                               Z e%e?j                  j                         e%e?j                  j                         e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        dAe#dee:eJ      dee:eO      de#fd                                                               ZddZ e%e?j                  j                  e?j                  j                  g      e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                         e1dd      	 ddAe#de:eJ   deeO   de#fd                            Z e%e?j                  j                  e?j                  j                  g      e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                         e1dd      	 ddAe#de:eJ   deeO   de#fd                            Z e%e?j                  j                  e?j                  j                  g      e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                         e1dd      	 	 ddAe#de:eJ   deeO   deeO   de#f
d                            Z e%e?j                  j                  e?j                  j                  g      e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                         e1dd      	 	 ddAe#de:eJ   deeO   deeO   de#f
d                            Z e%e?j                  j                  e?j                  j                  g      e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                         e1dd      	 	 	 ddAe#de:eJ   deeO   deeO   deeO   de#fd                            Z e%e?j                  j                  e?j                  j                  g      e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                         e1dd      	 	 	 ddAe#de:eJ   deeO   deeO   deeO   de#fd                            ZeF	 ddAe#de:eJ   de:eeO      deBde#f
d       Zd Zd Zd  Zd! Z	 dd"Zd# Zd$ Zdd%Zdd&Zd' Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d(                      Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d)                      Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d*                      Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d+                      Zd, Zdd-Zdd.Z d/ Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d0                      Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d1                      Zd2 Zd3 Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d4                      Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d5                      Z	 e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d6                      Z e%e?j                  j                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d7                      Z e%e?j                  j                         e%e?j                  j                        e?j                   j                  j#                  e8j$                        e?j                   j                  j#                  e8j&                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        e?j                  j                  j#                  e8j$                        e?j                  j                  j#                  e8j&                        d8                                                         Z e%e?j                   j                  e?j                   j                  g       e1       	 ddAe#de:eJ   d9eBdeeO   de#f
d:              Z e%e?j                  j                  e?j                  j                  g      e?j                  j                  j#                  e8j&                         e1       	 	 ddAe#de:eJ   d9eBdeeO   deeO   de#fd;                     Z e%e?j                  j                  e?j                  j                  g       e1       	 	 	 ddAe#de:eJ   d9eBdeeO   deeO   deeO   de#fd<              Zdd=Zd> Zdwee#   d?ee#   d@e#de#fdAZd?e,de#fdBZeFdAe#de:eJ   d9eBde:eeO      de#f
dC       Z e%e?j.                  j                        dDe#dEe#deBfdF       Z e%e?j0                  e?j2                  g       e1       dG               Z e%e?j4                  g      dH        Z e%e?j6                  g      ddI       Z e%e?j8                  g      dJ        Z e%e?j:                  g      dK        Zd2e#dTe#dEee#   dPeJd_eJdebe#e#f   fdLZ e%e?j>                         e1dd`      d2e#dTe#dEee#   dPeJd_eJdebe#e#f   fdM              Z e%e?j@                         e1dd`      d2e#dTe#dEee#   dPeJd_eJdebe#e#f   fdN              Z de#dOeOde#fdPZ!de#dOeOde#fdQZ"dRe#de,fdSZ#dTe,dUe#de#fdVZ$dUee#   de#fdWZ%dXeJd9eBdRej                  dej                  fdYZ&dZe#d[eJd\eJd9eBfd]Z'dZe#d^eJd[eJd\eJd9eBf
d_Z(dZe#de:eJ   d9eBfd`Z)dZe#de:eJ   d9eBfdaZ* e%e?jV                         e1       eFdZe#de:eJ   d9eBfdb                     Z+	 	 	 	 ddDe#dce#ddeJdeeJd9eBdfeBde#fdgZ, e%e?jZ                         e1       eF	 	 	 ddDe#dce#ddeJdeeJd9eBde#fdh                     Z- e%e?j\                         e1d      eFdi                      Z. e%e?j^                         e1       dde@j                  j                  fdj              Z/dkejF                  dlejF                  dmeBdeBfdnZ0e?jb                  j                  j#                  e8j$                        e?jb                  j                  j#                  e8j$                         e1do      ddpdq                     Z1 e%e?jd                  j                  e?jd                  j                  g      e?jd                  j                  j#                  e8j&                         e1       eF	 	 ddAe#debeJeJf   d9eBdreeO   dseeO   de#fdt                            Z3 e%e?jd                  j                        e?jd                  j                  j#                  e8j$                        e?jd                  j                  j#                  e8j&                         e1       eF	 ddDe#deebeJeJf      d9eBdeebeOeOf      de#f
du                                   Z4 e%e?jj                         e%e?jl                         e%e?jn                        eF e1       dDe#debeJdf   de#fdv                                   Z8 e%e?jr                         e%e?jt                         e%e?jv                        eF e1       dDe#debeJdf   de#fdw                                   Z<dDe#debeJdf   dxeeJeJeJge#f   de#fdyZ= e%e?j|                         e%e?j~                         e%e?j                         e1d       dz                             ZA e%e?j                         e1d{d|      ddd}d~              ZB e%e?j                         e1       dddd              ZC e%e?j                  j                  e?j                  j                  g       e1       dej                  ddddqe)dReej                     dej                  deej                     deBf
d              ZG e%e?j                  j                  g      dej                  ddddpe)dqe)dReej                     dej                  deej                     deBfd       ZI e%e'      d        ZJ e%e?j                        e?j                  j                  j#                  e8j&                         e1       dtdtde@j                  j                  fdAe#dTe#dje)de)dEee#   dPeJde#fd                     ZK e%e?j                        e?j                  j                  j#                  e8j&                         e1dd      dAe#dTe#dPeJdebe#e#f   fd                     ZL e%e?j                  j                        	 	 ddddde#de#d0e#deOdeBdee#   d*eeO   debe#e#f   fd       ZNd ZO e%e?j                  g       e1d      eFdd                     ZP e%e?j                         e1       d               ZQ e%e?j                        d        ZR e%e?j                  j                  e?j                  j                  g      dddd2e#dReej                     dee#   de#fd       ZT e%e?j                  j                  e?j                  j                  g      dd2e#deeJ   fd       ZW e%ej|                  j~                  j                        dd       ZX e%e?j                         e1       dddd              ZY e%e?j                  j                        ddd2ejF                  deej                     dejF                  fd       ZZdddZ[ddddZ\ e%e?j                         e1       d               Z] e%e?j                        dd       Z^ eOe?j                  e?j                          eOe?j                  e?jL                          eOe?j                  e?jP                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                           eOe?j                  e?j                          eOe?j                  e?j                          eOe?j
                  e?j                          eOe?j                  e?j                          eOe?j                  e?j                         y(      N)CallableIterable)nullcontext)Enum)partialreduce)chainproduct)AnycastOptionalUnion)	sym_floatsym_intTensorregister_decomposition)	out_dtype)IntLike
NumberTypesuggest_memory_format
TensorLikeTensorSequenceType)_maybe_convert_to_dtype_maybe_resize_out_safe_copy_outout_wrapper)_pytree)tree_map__all__c                       e Zd ZdZdZdZy)	Reductionr         N)__name__
__module____qualname__NONEMEANSUM     V/var/www/html/engine/venv/lib/python3.12/site-packages/torch/_decomp/decompositions.pyr"   r"   1   s    DD
Cr,   r"   Fftype_promotioncompute_dtype_onlyinclude_non_tensor_argsc                 N     t        j                          fd       }|S )Nc                  \  	 r t         t        j                  j                  fnt         f}t	        j
                  | i |D cg c]  }t        ||      r| }}t        j                  |di\  	fd}	fd} t        ||       i t        ||      }
r|S t        ||      S c c}w )Ntype_promotion_kindc                 J    t        | t              r| j                        S | S N
isinstancer   to)xcomputation_dtypes    r-   increase_precz0type_casts.<locals>.inner.<locals>.increase_precO   s"    !V$tt-..r,   c                 J    t        | t              r| j                        S | S r6   r7   )r:   result_dtypes    r-   decrease_precz0type_casts.<locals>.inner.<locals>.decrease_precU   s!    !V$ttL))r,   )
r   torchtypes_Numberpytreearg_tree_leavesr8   utilselementwise_dtypesr   )argskwargsallowed_typesr:   	flat_argsr<   r?   rr;   r>   r0   r.   r1   r/   s           @@r-   innerztype_casts.<locals>.inner@   s     .EVU[[(()6) 	
 ++T<V<
!]+ 
	 

 +0*B*B+
,:+
'<
		 xt,P0OPHM1--5
s   B))	functoolswraps)r.   r/   r0   r1   rL   s   ```` r-   
type_castsrO   :   s(     __Q. .@ Lr,   T)r/   r0   )r/   )r/   r1   r:   dimreturnc                 j    t        || j                         z
        D ]  }| j                  d      }  | S N)rangerP   	unsqueeze)r:   rP   _s      r-   _unsqueeze_to_dimrX   w   s2    3=! KKOHr,   
grad_inputout_gradyc                 4    | d||z  z
  j                         z  S Nr#   conj_physicalrZ   r[   s     r-   tanh_backwardra   }   s      q1q5y//111r,   c                 4    | |d|z
  z  j                         z  S r]   r^   r`   s     r-   sigmoid_backwardrc      s      qAE{11333r,   beta	thresholdc                 t    ||z  j                         }t        j                  ||z  |kD  | | |z  |dz   z        S N      ?)expr@   where)rZ   r:   rd   re   zs        r-   softplus_backwardrl      s=     
TA;;DI-xAS9QRRr,   grad_outputalphascaleinput_scale	is_resultself_or_resultc                     ||z  }|}|}|r&t        j                  |dk  | |z  ||z   z  | |z        S t        j                  |dk  | |z  |z  t        j                  ||z        z  | |z        S Nr   )r@   rj   ri   )	rm   rn   ro   rp   rq   rr   negcoefposcoef
negiptcoefs	            r-   elu_backwardrx      s     emGGJ{{a*$(@A'!
 	
 {{a*$w.>J;V1WW'!
 	
r,   c                 .    t        j                  | |      S r6   )r@   	full_likeselfvalues     r-   fill_scalarr~      s    ??4''r,   r}   c                     t        j                  j                         dk(  fd       t        j	                  |       S )Nr   c                  ,    d j                          dS )Nz@fill only supports 0-dimension value tensor but got tensor with z dimensionsrP   )r}   s   r-   <lambda>zfill_tensor.<locals>.<lambda>   s    RSXS\S\S^R__jk r,   )r@   _checkrP   atencopyr{   s    `r-   fill_tensorr      s3    	LL		qk 99T5!!r,   r|   c                 f    t        j                  t        j                  | dz   d      d      dz  S N   r   min   maxr@   clampr|   s    r-   hardsigmoidr      s)     ;;u{{4!83;a??r,   c                 H    t        j                  |dkD  |dk  z  | dz  d      S )Ng      g      @gUUUUUU?        r@   rj   rm   r|   s     r-   hardsigmoid_backwardr      s0     ;;	$y! r,   min_valmax_valc                 B    t        j                  ||k  ||k\  z  d|       S )Nr   r   )rm   r|   r   r   s       r-   hardtanh_backwardr      s$    
 ;;DGO<c;OOr,   c                 l    | t        j                  t        j                  | dz   d      d      z  dz  S r   r   r   s    r-   	hardswishr      s.     %++ekk$(:BBQFFr,   c           
      x    t        j                  |dk  dt        j                  |dk  | |dz  dz   z  |             S )Nr   r         ?r   r   s     r-   hardswish_backwardr      sA     ;;
D1HkdQh#-=>L r,   c                 6    t        j                  ||k  d|       S rt   r   )rm   r|   re   s      r-   threshold_backwardr      s     ;;ty(![99r,   negative_slopeself_is_resultc                 <    t        j                  |dkD  | | |z        S rt   r   )rm   r|   r   r   s       r-   leaky_relu_backwardr      s      ;;taxkN.JKKr,   gradapproximatec                    d}d}d}|dk(  ri||z  dz  }d}||z  }||z  }	||||	z  z   z  }
t        j                  |
      }d|z  }d|z   }d|z  }d||z  z
  }|dd|z  |z  z   z  }||z  |z  }| ||z   z  S |}||z  dz  }ddt        j                  ||z        z   z  }|t        j                  ||z  d	z        z  }| |||z  z   z  S )
Ng;f?g;f?gmBP?tanhr   gHm?r#   r   g      )r@   r   erfri   )r   r|   r   M_SQRT2	M_SQRT1_2
M_2_SQRTPIkBetakKappax_sqx_cuberL   
tanh_innerleftrightleft_derivativetanh_derivativeinner_derivativeright_derivativekAlphacdfpdfs                        r-   gelu_backwardr      s'    %G&I'Jf*$s*d{/0ZZ&
TzJ+j:55 AF
T(9$9:/14DD)99::Y&,Q4&=112eiitd 233sTCZ'((r,   inputc                     t        j                  t        j                  |            }t        j                  |      }||z  d||z  z
  z  }| ||z   z  S r]   )r@   r   Fsoftplussigmoid)rm   r   input_tanh_softplusinput_sigmoidouts        r-   mish_backwardr     sV      **QZZ%67MM%(M
-
1':=P'P#P
QC-344r,   c                 2    | t        j                  |       z  S r6   )r@   r   r   s    r-   silur   "  s     %--%%%r,   c                 \    ddt        j                  |       z   z  }| |z  d|d|z
  z  z   z  S r]   )r@   ri   )rm   r|   r   s      r-   silu_backwardr   )  s<     1uyy$''(G AG(<$<==r,   weightc                 <    t        j                  | dkD  | || z        S rt   r   )r|   r   s     r-   _prelu_kernelr   1  s    ;;taxv}55r,   c                 ~    t        j                  |dkD  | || z        }t        j                  |dkD  d|| z        }||fS )Nr   r   r   )rm   r|   r   
input_gradweight_grads        r-   _prelu_kernel_backwardr   6  sE     TAX{F[4HIJ++dQhTK-?@K$$r,   noiseloweruppertrainingc                 x    |r||z
  dkD  r| j                  |      S ||z   dz  }t        j                  | |||      S )Ngư>r$   )mulr   r   )rm   r|   r   r   r   r   r   r   s           r-   rrelu_with_noise_backwardr   A  sK     EEMD(u%%%-1,''~~
 	
r,   bufferc                     |dk  }t        j                  |dd      }t        j                  |dd      }t        j                  t        j                  |             }| |||d|z   z  z  z
  z  S )Nr   r#   rT   )r@   rj   ri   abs)rm   r|   r   in_negative	max_derivsignrk   s          r-   log_sigmoid_backwardr   V  sg     (KKA.I;;{Ar*D		599T?"#A)da1q5k&::;;r,   loss	reductionc                     |t         j                  j                  k(  rt        j                  |       S |t         j
                  j                  k(  rt        j                  |       S | S r6   )r"   r)   r}   r@   meanr*   sum)r   r   s     r-   apply_loss_reductionr   c  sH    INN(((zz$	imm))	)yyr,   dtypec                     | t         j                  k(  rt         j                  S | t         j                  k(  rt         j                  S | t         j
                  k(  rt         j                  S y r6   )r@   	complex32float16	complex64float32
complex128float64r   s    r-   to_real_dtyper   l  sK    }}	%//	!}}	%""	"}} 
#r,   targetc                 *    | |z
  dz  }t        ||      S )Nr$   )r   )r|   r   r   r   s       r-   mse_lossr   {  s     6MaDi00r,   c                 |    |t         j                  j                  k(  rd|j                         z  nd}|||z
  z  | z  S )N       @)r"   r)   r}   numel)rm   r   r   r   norms        r-   mse_loss_backwardr     s;     #,y~~/C/C"C3D56>"[00r,   c                     t        j                  | ||      }| j                  t        d            }t        j                  ||d      }t        j
                  |      }t        j                  |||      S )N)rP   r   z-infTrP   keepdim)r@   softmaxeqfloatall
zeros_likerj   )r|   rP   r   r   maskedmasked_rowszeross          r-   safe_softmaxr    s[    
--#U
3CWWU6]#F))FT:KS!E;;{E3//r,   rh   c                     | |z
  j                         }t        j                  ||k  d|dz  z  |z  |d|z  z
        }t        ||      S )Nr   r$   )r   r@   rj   r   )r|   r   r   rd   r   s        r-   smooth_l1_lossr    sO     6M D;;td{C$'MD$8$t:KLDi00r,   c                    |t         j                  j                  k(  rd|j                         z  nd}||z
  }t	        j
                  |      }|| z  }t	        j                  ||k  ||z  |z  |t	        j                  |      z        S rg   )r"   r)   r}   r   r@   r   rj   r   )	rm   r|   r   r   rd   r   r:   abs_x	norm_grads	            r-   smooth_l1_loss_backwardr
    s{    
 "+inn.B.B!B3DvAIIaLE{"I;;AEJJqM! r,   c                 h    t        | ||||      }t        ||j                         t        ||d      S NT	copy_fromcopy_toexact_dtype)r
  r   shaper   )rm   r|   r   r   rd   rY   results          r-   smooth_l1_loss_backward_outr    s3     %[$	4PFj&,,/FJDQQr,   deltac           
          |t         j                  j                  k(  rd|j                         z  nd}||z
  }t	        j
                  || k  | | z  |z  t	        j
                  ||kD  || z  |z  ||z  | z              S rg   )r"   r)   r}   r   r@   rj   )rm   r|   r   r   r  r   r:   s          r-   huber_loss_backwardr    s    
 "+inn.B.B!B3DvA;;	UF
	e#AItk1E94!8k;QR r,   c                 h    t        | ||||      }t        ||j                         t        ||d      S r  )r  r   r  r   )rm   r|   r   r   r  rY   r  s          r-   huber_loss_backward_outr    s3     !dFIuMFj&,,/FJDQQr,   ignore_indextotal_weightc                    |j                         dk  rdnd}|t        j                  j                  k(  r| |z  } |j	                  |      }t        j                  ||k7  |d      }t        j                  |      }	t        j                  |	||d      }	|	j                         | j                         cxkD  rdkD  rn n| j	                  |      } |Nt        |j                               D 
cg c]  }
d }}
|j                  d   ||<   |j                  |      }| |z  } t        j                  ||k7  | d      } |	| z  S c c}
w )Nr$   r   r#   g      )rP   r"   r)   r}   rV   r@   rj   r   scatterrU   r  reshape)rm   r|   r   r   r   r  r  channel_dimsafe_targetrY   rW   	new_shapes               r-   _nll_loss_backwardr!    s    xxzA~!1KINN(((!L0k*F++f4fa@K!!$'Jz;TJJ~~+//+/a/!++K8 %dhhj 121Q2	2!'a	+	*!F*++f4k1EK## 3s   *	D<c                    |j                         dkD  sJ d       t        j                  |j                         |      }|j                  |      }|dz  dk(  sJ d| d|        |dz  }|j	                  |d|      }|j	                  |||      }t        j                  |      }d|z
  |z  |z  | z  }	|| z  }t        j                  ||	g|      S )Nr   z*glu does not support 0-dimensional tensorsr$   z.Halving dimension must be even, but dimension z	 is size rh   r   )rP   rE   canonicalize_dimsizenarrowr@   r   cat)
rm   r|   rP   wrap_dimnIn	inputSize	firstHalf
secondHalfgradInputFirstHalfgradInputSecondHalfs
             r-   glu_backwardr.    s     88:>GGG>%%dhhj#6H
))H
C7a< 
8
)C5Q< qIHa3IXy)<Jz2	!	!%77)CkQ  ,k999(*=>HMMr,   c           	         d|j                         cxk  rdk  sJ d        J d       |j                         dk  sJ d       |j                         dk(  xr |j                         dk(  }|sA|j                  d   |j                  d   k(  s"J d|j                   d|j                   d       |j                         dk(  s'J d	|j                   d
|j                          df       |'|j                         |j                  d   k(  sJ d       |t        j                  j
                  k(  r}|j                         dk(  rj| j                         dk(  r| j                  d   |j                  d   k(  srJ d|j                  d    d| j                          d| j                  d           | j                         dk  r| j                         dk(  sJ d| j                          t        | ||||||      S )Nr   r$   input tensor should be 1D or 2Dr#   ;0D or 1D target tensor expected, multi-target not supportedsize mismatch (got input: 
, target: ):expected total_weight to be a single element tensor, got: z (z
 elements)rT   z<weight tensor should be defined either for all or no classesz7Expected a tensor of dimension 1 and tensor.size[0] == z but got: dimension z and tensor.size[0] == z7Expected a single element grad_output tensor, but got: )rP   r  r   r"   r(   r}   r!  )rm   r|   r   r   r   r  r  no_batch_dims           r-   nll_loss_backwardr7    s    
aB!BBB!BB::<1 E 88:?8vzz|q'8LDJJqMV\\!_< 
$TZZL
6<<.J= 1$ D
b!3!3!5 6jA' $
 >V\\^tzz"~= F= INN(((TXXZ1_ A%+*;*;A*>$**Q-*O 	
EdjjQRm_ U)oo/00GHYHYZ[H\G]_	
O
  A%+*;*;*=*B 	
EkFWFWEXY	
B T669lL r,   c           	      >   |j                         dk(  sJ d|j                                 |j                         dk(  sJ d|j                                 |j                  d   |j                  d   k(  r>|j                  d   |j                  d   k(  r|j                  d   |j                  d   k(  s!J d|j                   d	|j                          |j                         dk(  s&J d
|j                   d|j                          d       t        | ||||||      S )N   zSonly batches of spatial inputs supported (4D tensors), but got input of dimension: r   zUonly batches of spatial targets supported (3D tensors) but got targets of dimension: r   r$   r#   r2  r3  r5  z ( z, elements))rP   r  r   r!  )rm   r|   r   r   r   r  r  s          r-   nll_loss2d_backwardr:  F  s@    88:? 
]^b^f^f^h]ij? ::<1 
_`f`j`j`l_mn
 	

1a(JJqMV\\!_,JJqMV\\!_,I 
$DJJ<z&,,H	I	-
 1$ 	""#3|'9'9';&<K	I$
 T669lL r,   c           	      "   |dz
  t        j                  t        j                  |        | j                  dd            z  |t        j                  t        j                  |       | j                  dd            z  z
  }|||z  }t        ||      S )Nr#   r+   i)r@   maximumlog1pnew_fulllogr   )r|   r   r   r   r   s        r-   binary_cross_entropyr@  i  s     QJ%--TEDMM"d3 uyyb$0GHHID f}i00r,   c                     d}| ||z
  z  t        j                  |d|z
  z  |      z  }|||z  }|t        j                  j                  k(  r||j                         z  }|S )Ng-q=r#   r   )r@   r   r"   r)   r}   r   )rm   r|   r   r   r   EPSILONr  s          r-   binary_cross_entropy_backwardrC    sg     GD6M*U[[T9JPW-XXF&INN((($**,&Mr,   c                 r    t        j                  t        j                  |  |z              }t        ||      S r6   )r@   r=  ri   r   )r   r   r   r   s       r-   soft_margin_lossrE    s.     ;;uyy%&12Di00r,   c                     || z  t        j                  ||z        dz
  z  }|t        j                  j                  k(  r||j                         z  }|S r]   )r@   r   r"   r)   r}   r   )rm   r|   r   r   rY   s        r-   soft_margin_loss_backwardrG    sM     +%v})E)IJJINN((($**,.
r,   otherpc                 6    t         j                  | |z
  |      S )N)rI  )r   r   )r   rH  rI  s      r-   distrK    s     99UU]a9((r,   x1x2c                    | j                  d      j                  dd      }t        j                  |t        j                        }|j                  d      j                  dd      }t        j                  |t        j                        }t        j
                  | j                  d      ||gd      }t        j
                  |||gd      }|j                  |j                        }|j                  d      j                         S )Nr$   rT   Tmemory_formatr   )powr   r@   	ones_likecontiguous_formatr&  r   matmulmT	clamp_minsqrt)	rL  rM  x1_normx1_padx2_normx2_padx1_x2_r  s	            r-   _euclidean_distr_    s     ffQimmB%G__WE4K4KLFffQimmB%G__WE4K4KLF
))RVVBZ&12
6C
))R)2
.CZZFA##%%r,   input_sizesstartendstepc                 X    | j                  |      }t        j                  || ||||      S r6   )	new_zerosr@   slice_scatter)rm   r`  rP   ra  rb  rc  rY   s          r-   slice_backwardrg    s/     &&{3Jz;UCNNr,   r#   c                    ddl m} | j                         }|dk(  rt        d      t	        j
                  | j                         |      }t        | j                               }t        | j                               }|dk  rt        d      ||nd}	||nt        j                  }
|	dk  r|	||   z  }	|
dk  r|
||   z  }
|	dk  rd}	n|	||   kD  r||   }	 ||
t        j                  k(        r||   }
n|
|	k  r|	}
n|
||   kD  r||   }
| j                         |	||   z  z   }|
|	z
  }||z   dz
  |z  ||<   ||xx   |z  cc<   | j                  rt        d      | j                  |||      S )Nr   statically_known_truez,slice() cannot be applied to a 0-dim tensor.zslice step must be positiver#   z<Slice decomposition for quantized tensors aren't implemented)%torch.fx.experimental.symbolic_shapesrj  rP   RuntimeErrorrE   r#  listr$  stridesysmaxsizestorage_offsetis_quantizedNotImplementedError
as_strided)r|   rP   ra  rb  rc  rj  ndimsizesstrides	start_valend_valrq  lens                r-   slice_forwardr{    s    L88:DqyIJJ

 
 S
1CE4;;=!Gqy899*I_c#++G1}U3Z	{5:1}		U3Z	#J	W34*	9		5:	*((*Y-EEN
I
C*q.T)E#JCLDL!J
 	
 ug~>>r,   c                 n    | j                   |   dt        ffd} ||dd      } |||      }||fS )zn
    Normalize start and end such that both are in the range
    [0, x.get_size()[dim]] and start <= end.
    rQ   c                 L    | |S | dk  r| z   } t        t        | |      |      S rt   r   r   )valr   r   defaultdim_sizes       r-   
clamp_wrapz(_normalize_start_end.<locals>.clamp_wrap  s0    ;N7.C3sE?E**r,   r   )r  int)r:   rP   ra  rb  r  r  s        @r-   _normalize_start_endr    sJ     wws|H+# + ua1-E
S%8
4C#:r,   srcc           	      |   t        j                  | j                  |      }| j                  |   }t	        | |||      \  }}t        | j                        }||z
  |dz
  z   |z  ||<   |j                  |      }|dk(  r||k(  r|dk(  r|j                         S d g| j                         z  }t        j                  || j                        }	|	|z
  |z  ||<   t        j                  || j                  t        j                        }
|dk7  rt        j                  |
|	|k\        }
||k7  rt        j                  |
|	|k        }
|dk7  rt        j                  |
|	|z
  |z  dk(        }
dg| j                         z  }d||<   |
j                  |      }
t         j#                  |
t         j%                  ||
|d      |       S )Nr#   r   devicer  r   rT   )rE   r#  ru  r  r  rm  expandclonerP   r@   aranger  onesboollogical_andviewr   rj   _unsafe_masked_index)r   r  rP   ra  rb  rc  r  src_sizeindicesidxmask
mask_shapes               r-   rf  rf    s    
 
 S
1C{{3H%eS%=JE3EKK H5[D1H-$6HSM
**X
CzcXo$!)yy{'+fuyy{&:G
,,x
5C%KD(GCL::hu||5::FDz  se|4
h  sSy1qy  et';q'@Auyy{"JJsO99Z D::dD55c4!LeTTr,   indexc                 T    | j                  |      }t        j                  || ||      S r6   )re  r@   select_scatter)rm   r`  rP   r  rY   s        r-   select_backwardr  E  s+     &&{3J
KeDDr,   offsetdim1dim2c                 V    | j                  |      }t        j                  || |||      S r6   )re  r@   diagonal_scatter)rm   r`  r  r  r  rY   s         r-   diagonal_backwardr  L  s-    
 &&{3J!!*k64NNr,   input_dtypec                 F    | j                   |k7  r|j                  |      }|S r6   )r   r9   )rm   rY   r  s      r-   _cast_grad_to_input_dtyper  U  s&     K']];/
r,   outputc                 ~    | |z  }||t        j                  ||d      z  z
  }t        | ||      j                         S NTr   )r@   r   r  
contiguous)rm   r  rP   r  new_grad_outputrY   s         r-   _softmax_backward_datar  ]  sK     "F*O 6EIIS$- $ J %[*kJUUWWr,   c                 ~    | t        j                  |      t        j                  | |d      z  z
  }t        | ||      S r  )r@   ri   r   r  )rm   r  rP   r  rY   s        r-   _log_softmax_backward_datar  o  sA     uyy0599d4   J %[*kJJr,   c                     | |dz  z   ||dz
  z  z
  }t        t        j                  t        j                  |      } |d||      j	                  d      } |d||z  |      j	                  d      }	||	z   S )z/Utility function to implement im2col and col2imr$   r#   r   r  r   rT   )r   r@   r  int64rV   )
input_dkernel_d
dilation_d	padding_dstride_dr  blocks_d	arange_kwblocks_d_indiceskernel_grids
             r-    _im2col_col2im_indices_along_dimr  {  s     Q&x!|)DDHEKKGI !Hh7AA!D Ax*4jAKKBOK k))r,   kernel_sizedilationpaddingrn  c           
         t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        dd} |d        |d        |d	d
        |d       | j                  t              }t        j                  |dv xr t	        d dd  D              fd       t        d t        dd        D              t        j                  t	        d D              fd       |dk(  }|s| j                  d      } | j                  \  }}	}
}\  }}\  }}\  }}\  }}t        |
||||| j                        }t        |||||| j                        }t        j                  | ||||f      }|j                  d      j                  d      }|d d d d ||f   }|j                  dddddd      }|j                  d      }|j                  d      }|j                  ||	|z  |z  ||z        }|s|j                  d      }|S )Nr$   c                       y)Nz"im2col(): only 2D kernel supportedr+   r+   r,   r-   r   zim2col.<locals>.<lambda>      r,   c                       y)Nz$im2col(): only 2D dilation supportedr+   r+   r,   r-   r   zim2col.<locals>.<lambda>  r  r,   c                       y)Nz#im2col(): only 2D padding supportedr+   r+   r,   r-   r   zim2col.<locals>.<lambda>  r  r,   c                       y)Nz"im2col(): only 2D stride supportedr+   r+   r,   r-   r   zim2col.<locals>.<lambda>  r  r,   c                      |rt        d  D              nt        d  D              }t        j                  | fd       y )Nc              3   &   K   | ]	  }|d kD    ywr   Nr+   .0rI  s     r-   	<genexpr>z1im2col.<locals>.check_positive.<locals>.<genexpr>       (Q1q5(   c              3   &   K   | ]	  }|d k\    ywr  r+   r  s     r-   r  z1im2col.<locals>.check_positive.<locals>.<genexpr>       ;RqAF;Rr  c                       d  S Nz& should be greater than zero, but got r+   param
param_names   r-   r   z0im2col.<locals>.check_positive.<locals>.<lambda>      ZL(NugV r,   r   r@   r   r  r  strictconds   ``  r-   check_positivezim2col.<locals>.check_positive  4    ,2s(%((;RE;R8RV	
r,   r  r  r  Fr  rn  r   r9  c              3   &   K   | ]	  }|d k7    ywr  r+   r  ds     r-   r  zim2col.<locals>.<genexpr>       :!qAv:r  r   c                       dt                S )NzmExpected 3D or 4D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: tupler  s   r-   r   zim2col.<locals>.<lambda>       --25\N< r,   c              3   \   K   | ]$  \  }}}}}d |d|z  z   ||d z
  z  z
  d z
  |z  z    & yw)r#   r$   Nr+   )r  r   paddilkersts         r-   r  zim2col.<locals>.<genexpr>  sF      "Cc3 	
S1s7]SC!G_,q0R77s   *,rQ  c              3   &   K   | ]	  }|d kD    ywr  r+   )r  cs     r-   r  zim2col.<locals>.<genexpr>  s     'aAE'r  c                  F    dt        dd         d d  d d d dS )	Nz!Given an input with spatial size rQ  , kernel_size=, dilation=
, padding=	, stride=z9, the calculated shape of the array of sliding blocks is z*, but its components must be at least one.r  )r  r  output_sizer  r  rn  s   r-   r   zim2col.<locals>.<lambda>  sL    3E%*4E3F G"m;xj 9)9VH -]D	F r,   r9  r   rT   r#   r      T)r@   r   rz  r  r   r  ziprV   r  r  r   r  permuter$  r  squeeze)r   r  r  r  rn  r  ru  batched_input	batch_dimr  input_hinput_wstride_hstride_w	padding_h	padding_w
dilation_h
dilation_wkernel_hkernel_wblocks_row_indicesblocks_col_indicespadded_inputr  num_blocks_rownum_blocks_colr  r  s    ````                     @@r-   im2colr    s~    
LL[!Q&(TU	LLX!#%ST	LLW"$QR	LLV!#OP
 ;.8Z(8Yu568$KKEu:D	LL:3:uRSz::	<
  &)"#J;'
 K 
LL';''	F 	F AIM"/4{{,I{GWHh"Iy%J
$Hh9:y(ELL ::y(ELL 55Iy) LML+55b9CCBG!Q 24FFGF^^Aq!Q1-F',,Q/N',,Q/N^^;)H4n~6UF "Mr,   r  c                   !" t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        t        j                  t              dk(  d        dd} |d	        |d
        |dd        |d        |d       | j                  "t        "      }t        j                  |dv xr t	        d "dd  D              "fd       d   d   z  }t        j                  "d   |z  dk(  "fd       t              D 	
cg c]"  \  }	}
}}}d|	d|
z  z   ||dz
  z  z
  dz
  |z  z   $ }}}}
}	}|d   |d   z  !t        j                  "d   !k(  !"fd       t        j                  !dkD  !"fd       |dk(  }|s| j                  d      } | j                  "\  }}\  }}\  }}\  }}\  }}| j                  "d   "d   |z  gz   |z         } | j                  dddddd      } t        |||||| j                        }t        |d      }t        |||||| j                        }t              D cg c]  \  }}|d|z  z    }}}| j                  "d   "d   t              z  g|z         }d d ||f} t        j                  || | d      }t!        j"                  || | | | f      }|s|j%                  d      }|S c c}}}}
}	w c c}}w )Nr$   c                       y)Nzonly 2D output_size supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  r  r,   c                       y)Nzonly 2D kernel supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  r  r,   c                       y)Nzonly 2D dilation supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  r  r,   c                       y)Nzonly 2D padding supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  r  r,   c                       y)Nzonly 2D stride supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  r  r,   Tc                      |rt        d  D              nt        d  D              }t        j                  | fd       y )Nc              3   &   K   | ]	  }|d kD    ywr  r+   r  s     r-   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r  c              3   &   K   | ]	  }|d k\    ywr  r+   r  s     r-   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r  c                       d  S r  r+   r  s   r-   r   z0col2im.<locals>.check_positive.<locals>.<lambda>  r  r,   r  r  s   ``  r-   r  zcol2im.<locals>.check_positive  r  r,   r  r  r  Fr  rn  r  )r$   r   c              3   &   K   | ]	  }|d k7    ywr  r+   r  s     r-   r  zcol2im.<locals>.<genexpr>   r  r  rQ  c                       dt                S )NzmExpected 2D or 3D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: r  r  s   r-   r   zcol2im.<locals>.<lambda>  r  r,   r   r#   c                      dd    d  S )Nz|Expected size of input's first non-batch dimension to be divisible by the product of kernel_size, but got input.shape[-2] = rQ  z and kernel_size=r+   )r  r  s   r-   r   zcol2im.<locals>.<lambda>  s#     ==B2YK H"m% r,   rT   c                  :    d d d d d d  dd    d	S 
NzGiven output_size=r  r  r  r  z , expected input.size(-1) to be 	 but got rT   .r+   Lr  r  r  r  r  rn  s   r-   r   zcol2im.<locals>.<lambda>  F    $[M} M:Zy	& B))*9U2YKqB r,   c                  :    d d d d d d  dd    d	S r  r+   r  s   r-   r   zcol2im.<locals>.<lambda>  r  r,   r   r9  r  
accumulater  )r@   r   rz  r  r   r  rV   r  r  r  r  rX   re  prodr   _unsafe_index_putr   r  r  )#r   r  r  r  r  rn  r  ru  prod_kernel_sizer   r  r  r  r  colr  out_hout_wr  r  r  r  r  r  r  r   indices_rowindices_colorI  output_padded_sizer  r  r  r  s#    `````                           @@r-   col2imr(    s    
LL[!Q&(OP	LL[!Q&(JK	LLX!#%IJ	LLW"$GH	LLV!#EF
 ;.8Z(7Ie468$;.KKEu:D	LL:3:uRSz::	<
 #1~A6	LLb	$$)	% '*(K'
 "Cc3 	
S1s7]SC!G_,q0R77C  	AQA	LLb	Q	B 	B 
LL	A	B 	B AIM"KKELE5Hh"Iy%J
$Hh MM58U1X1A%AB[PSVVWEMM!Q1a+E2xY%,,K $K3K2xY%,,K 14K0IJ1!a!e)JJ__	q58tK0014FFF {K
0C##FC4#HFUU6YJ
YJ
KLF"MkV Ks   6'M9Mr  c                 z    | |j                  |       |z  z  j                  t        j                  |             }|S NrO  )type_asr  rE   r   )rm   r  ro   rK   s       r-   native_dropout_backwardr,  C  sB     
[1E9	:AA11+> 	B 	A Hr,   
input_size	dimensionr$  c                    t        |      dk(  rt        j                  | d      S t        j                  t        |      |      }t        j
                  ||   | j                  t        j                        }|j                  d||      j                         }| j                  d|dz         j                  ||dz         } | j                  |      }d|z  |fz   }t        j                  ||| d      j                         S )Nr   r  rT   r#   r6   Tr  )rz  r@   squeeze_copyrE   r#  r  r  int32unfoldflattenmovedimre  r   r  r  )	r   r-  r.  r$  rc  rP   r  rY   r  s	            r-   unfold_backwardr5  R  s    
 :!!!$**

 
 Z)
<C
,,z#t{{%++
NC
**Qd
#
+
+
-C<<C!G$,,S#':D 
+JcMSF"E!!*eTd!KVVXXr,   epsc           
      .   |A|}d|z
  }t        j                  t        j                  ||k\  ||k        | |d|z
  z  z  d      S t        j                  t        j                  |dk\  |dk        | |d|z
  z  z  |j                  dt	        d                  S )Nrh   r   r+   nan)r@   rj   r  r>  r   )rm   r|   r6  lohis        r-   logit_backwardr;  e  s    
 2X{{dbj$"*543:./
 	
 {{dck43;743:./MM"eEl+
 	
r,   trainc                 d    |r|dk7  rt         j                  | ||      d   S | j                         S rt   )r   native_dropoutr  )r   rI  r<  s      r-   dropoutr?  z  s3     a""5!U3A66{{}r,   out0out1c                    |r|dk7  r|dk(  r:t        j                  |       t        j                  | t         j                        fS | j                  j                  st        d      t        j                  |       |kD  }|| z  t        dd|z
  z        z  }||fS | t        j                  | t         j                        fS )Nr   r#   r   z?result type Float can't be cast to the desired output type Longrh   )	r@   r   r  r   is_floating_pointrl  	rand_liker   rS  )r   rI  r<  	bool_maskress        r-   r>  r>    s     a6$$U+U-=-=e5::-VWW{{,,Q  OOE*Q.	%%sQw"88YuuEJJ?@@r,   half_to_floatc                    ddl m} | j                         } |r| j                  t        j
                  k(  sJ t        j                  | t        j                  j                        \  }}| j                  |      }  || j                         dk(        rt	        j                  |       }n0t	        j                  | |d      }t	        j                  | |z
        }|t	        j                  ||d      z  }|s|j                  |      }|S Nr   guard_or_falser4   T)r   )rk  rK  r  r   r@   halfrE   rF   ELEMENTWISE_TYPE_PROMOTION_KINDDEFAULTr9   r   ri   amaxr   )	r:   rP   rG  rK  r;   r>   unnormalizedx_maxr  s	            r-   _softmaxrS    s     E 	
Aww%**$$$&+&>&>	uDDLL'#| 	
Aaggi1n%yy|

1c40yyU+EIIlCFFF<(Mr,   )r  c                    ddl m} | j                         } |r| j                  t        j
                  k(  sJ t        j                  | t        j                  j                        \  }}| j                  |      }  || j                         dk(        r| }nt	        j                  | |d      }| |z
  }t	        j                  t	        j                  t	        j                  |      |d            }||z
  }	|s|	j                  |      }	|	S rI  )rk  rK  r  r   r@   rM  rE   rF   rN  rO  r9   r   rP  r?  r   ri   )
r:   rP   rG  rK  r;   r>   shiftedrR  shifted_logsumexpr  s
             r-   _log_softmaxrW    s     E 	
Aww%**$$$&+&>&>	uDDLL'#| 	
Aaggi1n%

1c40e)		%))EIIg,>T"RS((F<(Mr,   r  padding_idxscale_grad_by_freqsparsec                     | j                         dk(  sJ d       |j                  dk  r4| j                  d|      }|j                  dk(  r|j                  d      }|S | |   S )Nr$   z'weight' must be 2-Dr#   r   )rP   ru  index_selectr  )r   r  rX  rY  rZ  r   s         r-   	embeddingr]    sd     ::<1444||q!!!W-<<1++a.C
gr,   num_weightsc                 t   t        j                  | t         j                  j                        \  }}| j	                  |      } t        |t        j                        }|rZ|j                  |f      }t        j                  |      }t        j                  ||g|d      }||   }	| |	j                  d      z  } t        ||k(  | j                        }
| j                  |
d      }| j                  |f| j                   |j                  d  z         }t        j                  ||g|d      j	                  |      S )NrL  Tr  rT   r   )rE   rF   rN  rO  r9   r   r@   longre  rS  r   r  rV   rX   ru  masked_fillr  )rm   r  r^  rX  rY  r;   r>   countsr  grad_weights_scaler  r   grad_weights                r-   embedding_dense_backwardre    s,    ',&>&>)N)N)V)V'#| ..!23K%guzz:G""K>2w'''	4D'Q#G_!$6$@$@$DDW3[5E5EFD""4+D''	**7<<>::K !!+y$4!PSS r,   c                 "    d}| D ]  }||z  }	 |S r]   r+   )r:   rK   is      r-   r  r    s$    	A 	QHr,   tensors
num_chunksc                 Z   g }| D ]  }|j                         }||   |z   dz
  |z  |z  }|||   k7  r;dgdz  |j                  |z
  dz
  z  d|||   z
  gz   }t        j                  ||d      }|d | t	        j
                  |dg      z   }|j                  |j                  |              |S )Nr#   r   r$   rT   )r$  ru  r   constant_pad_ndr@   Sizeappendr  )	rh  rP   ri  padded_tensorstensortensor_sizepad_along_dimr  	view_sizes	            r-   
_pad_chunkrs    s    
 N 9kkm$S)J6:zIJVK,,#'V[[3.23C 007 C ))&#q9F%

J3C(DD	fnnY789 r,   c                 R    | d   j                   }| D ]  }|j                   |k7  s y y)Nr   FTru  )rh  ru  ro  s      r-   have_same_ndimsrv    s2    1:??D ;;$ r,   c                     | d   j                         d | }| D ]-  }t        j                  |j                         d | |k(  d        / y )Nr   c                       y)NzG_chunk_cat expects same sizes of 0,...,dim-1 dimensions for all tensorsr+   r+   r,   r-   r   z+leading_dimension_matches.<locals>.<lambda>%  r  r,   )r$  r@   r   )rh  rP   leading_dim_sizesro  s       r-   leading_dimension_matchesrz     sN    
)$3/ 
KKM$3#44]	

r,   c                    t        j                  |dk\  d        t        j                  t        |       dkD  d        | d   j                  }| d   j                  }| D ]r  }t        j                  |j                         dkD  d        t        j                  |j                  |k(  d        t        j                  |j                  |k(  d        t t        |       r(t        j                  | d   j                         |      }nEt        j                  |dk\  d        | D ]&  }t        j                  ||j                  k  d	        ( t        | |       |S )
Nr#   c                       y)Nz&_chunk_cat expects positive num_chunksr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>.  r  r,   r   c                       y)Nz0_chunk_cat expects a non-empty input tensor listr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>0  r  r,   c                       y)Nz#_chunk_cat expects non-empty tensorr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>5  r  r,   c                       y)Nz8_chunk_cat expects all input tensors with the same dtyper+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>8  r  r,   c                       y)Nz8_chunk_cat expects all inputs tensors on the same devicer+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda><  r  r,   c                       y)NzK_chunk_cat expects non-negative dim when input tensors have different ndimsr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>C  r  r,   c                       y)Nz3_chunk_cat expects dim < ndim for all input tensorsr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>H  r  r,   )r@   r   rz  r   r  r   rv  rE   r#  rP   ru  rz  )rh  rP   ri  expected_dtypeexpected_devicero  s         r-   _preprocess_chunk_cat_inputsr  )  s-   
 
LLq"RS	LLGqT QZ%%Naj''O 	
V\\^a')VWLLN*N	
 	MM_,N	
	
 w$$WQZ^^%5s;1Ha	
  	FLLfkk!M	
 gs+Jr,   r   c                     t        | ||      }t        | ||      }|t        j                  ||dz         S t        j                  ||dz   |       |S )Nr#   )r   )r  rs  r@   r&  )rh  rP   ri  r   rn  s        r-   
_chunk_catr  N  sS     'wZ
@Cj9N
{yyq11		.#'s3
r,   split_sizesc                    t         j                  | ||      }|.|D cg c]"  }|j                  t        j                        $ c}S t        ||      D ])  \  }}t        ||j                         t        ||d       + y c c}w )Nr   rO  Tr  )	r   split_with_sizesr  r@   rT  r  r   r  r   )r|   r  rP   r   splitssr  splits           r-   split_with_sizes_copyr  _  s     ""4#">F
{HNO1e&=&=>OO f- 	NMFEfekk2UFM	N  Ps   'B
split_size.c                 D    t         j                  j                  | ||      S r6   )r   r  r   )r   r  rP   s      r-   unsafe_splitr  r  s    ::UJ44r,   c                 D    t         j                  j                  | ||      S r6   )r   r  r  )r   r  rP   s      r-   unsafe_split_with_sizesr  w  s       ((SAAr,   c                    | j                   }||   }|dk(  r|dk(  sJ | j                         fS ||z   dz
  |z  }ddlm}  ||      }t	        |      D cg c]  }| }}|||z  |z
  z
  |d<   t        j                  | ||      S c c}w )Nr   r#   )	guard_intrT   )r  detachrk  r  rU   r@   r  )	r|   r  rP   r`  r  chunksr  rg  r  s	            r-   r  r  ~  s    **K3HQ1}}#a'J6F @vF',V}5!:5K5 J$7($BCKO;;t[#.. 6s   	Btensor_indices_or_sectionsc                 p  	 |j                   j                  dk(  sJ |j                  t        j                  k(  sJ |j                         	t        j                  	dk(  xs 	dk(  	fd       	dk(  r4|j                         }t        |t              sJ | j                  ||      S t        }t        j                  j                         x}r|j                  x}r|j                  } |       5  |D cg c]  }|j                          }}d d d        | j                  |      S c c}w # 1 sw Y    xY w)Ncpur#   r   c                      d  dS )Nz{tensor_split expected tensor_indices_or_sections to be a zero-dimensional or one-dimensional tensor, but got a tensor with z dimsr+   )	split_dims   r-   r   zAtensor_split_tensor_indices_or_sections_py_impl.<locals>.<lambda>  s     <<E;eM r,   )r  typer   r@   r  rP   r   itemr8   r   tensor_splitr   _guardsdetect_fake_mode	shape_envignore_fresh_unbacked_symbols)
r|   r  rP   sectionsctx	fake_moder  rg  r  r  s
            @r-   /tensor_split_tensor_indices_or_sections_py_implr    s.    &,,11U:::%++u{{:::*..0I	LLQ()q.	M
 A~-224(G,,,  3//7799I9",,,I,99C
 U 	E)CDAqvvxDGD	E   #.. E	E 	Es   /D,4D'D,'D,,D5mat1mat2c                     | j                         s&| j                         st        |      }t        |      }|t        j                  ||      z  }|dk(  r|S ||| z  z   S rt   )rC  
is_complexr  r@   mm)r|   r  r  rd   rn   r   s         r-   addmmr    s]     !!#DOO,=4yE

%((4&
&Cqy
 r,   use_geluc                     t        | ||||      }|r8| j                  rt        j                  |d      S t        j                  |      S t        j	                  |      S )Nr   )r   )r  is_cudar   gelurelu)r|   r  r  rd   rn   r  r   s          r-   _addmm_activationr    sO     dD$
.C<<99Sf95599S>!99S>r,   vecc                     | j                         s&| j                         st        |      }t        |      }|t        j                  ||      z  }|dk(  r|S |j                         dk(  r|| z  S ||| z  z   S rt   )rC  r  r  r@   mvr   )r|   r  r  rd   rn   r   s         r-   addmvr    ss     !!#DOO,=4yE

%((4%
%Cqy

yy{ad{r,   r   rstdgammaNCHxWgroupoutput_maskc
           	      D   t        j                  | ||d       t        j                  || d       t        j                  |d       t        j                  |j                         z  z  k(  fd       t        j                  j                  fk(  fd       t        j                  d u xs j                         k(  fd       z  }
t        j                  |
z  k(  fd       t        j                  | |      j                        j                  dg      }| j                        j                  dg      }d }d }d }|	d	   r*d
|
z  z  }t        j                  |j                  d	            j                  |
      j                  d      }t        j                  |j                  d	            j                  |
      j                  d      }t        j                  |j                  d      j                  d|
            }n|j                  |
      j                  d      }|j                  |
      j                  d      }t        j                  |j                  d      t        j                  d|
f|j                              }|z  |z
  |z  |z  |z  |z  }| z  ||z  |z  z
  }|j                  d      }t        |d      }t        |d      }t        j                  | j                  |
      |      t        j                  |j                  |
      |      z   |z   }|j                  |j                        j                  |j                         }|	d   rk|j                  |
      |j                  |
      j                  d      z  z
  |j                  d      z  j                  d	g      j                        }|	d   r|j                  d	g      }|||fS )NF)allow_cpu_scalar_tensorsc                      d z  z   dS )NzExpect input to have z	 elementsr+   )r  r  r  s   r-   r   z,native_group_norm_backward.<locals>.<lambda>	  s    'A}I> r,   c                  .    d  d dj                    S )NzExpect mean to have shape (, z
, but got r  )r  r  r   s   r-   r   z,native_group_norm_backward.<locals>.<lambda>  s    -aS5'DJJ<P r,   c                  <    d  dj                          S d S )NzExpect gamma to have z elements but got rT   )r   )r  r  s   r-   r   z,native_group_norm_backward.<locals>.<lambda>  s-    's*<eN_U[[]<hi eg<hi r,   c                      d  d S )NzExpect number of channels z, to be evenly-divisible by number of groups r+   )r  r  s   r-   r   z,native_group_norm_backward.<locals>.<lambda>  s    ,QC/[\a[bc r,   r$   r   r   rh   rT   r#   r  r9  )rE   check_same_devicecheck_same_shaper@   r   r   r  r   r  r   rV   r  r  r  rX   r9   r   )rm   r   r   r  r  r  r  r  r  r  cpgdsdbd_inputd_gammad_biasr  ds_valdb_valc1c2c3s     ` `````             r-   native_group_norm_backwardr    s    
UD$ 
5+N	4F	LLQ$> 
LL

q%j P 
LL+!+i
 u*C	LL	S5[c 
;	&	+	+Aq#	6	:	:s	:	CB			!Q	$	(	(aS	(	1B $G $G#F1~39YYr5??1#56>>q%MQQRSTFYYr5??1#56>>q%MQQRSTFr"a,B
 ZZ5#.2215FZZ5#.2215Fr"

Auc?4;;?B tmf$,t3d:Q>S4Z&4-!++\\"r1%r1%IIk))!UC=rBiiaS92>? 	
 //%++.11%++>1~ E3'"''!UC*@4>>RTCU*UU..$% SaSS\WQZ 	 1~QCWf%%r,   out2c
                    t        | |||||||||	
      }|
||f}t        |      D ]2  \  }}|	t        ||   |j                         t	        |||   d       4 |S r  )r  	enumerater   r  r   )rm   r   r   r  r  r  r  r  r  r  r@  rA  r  r  rY   rg  rK   s                    r-   native_group_norm_backward_outr  M  sz    " (UD$q!S%F d#J&! Q1=jmQWW5Q
14PQ
 r,   c                 ,    | | j                  |      S | S r6   r9   )r:   r   s     r-   _maybe_castr  j  s    }ttE{Hr,   grad_outnormalized_shapebiasc                 Z  " |j                   }|j                         }	t        j                  |j                        ""fd| |||fD        \  }
}}}|
J |	t        |      z
  }||d  }|d | }g }g }t        |	      D ]*  }||k\  r|j                  |       |j                  |       , t        |      }t        |      }ddl	m
}  ||dk(        s ||dk(        rN|d   r|j                  |      nd |d   r|j                  ||d        nd |d   r|j                  ||d        fS d fS t        ||j                               }t        ||j                               }|J ||z
  |z  }||
|z  }n|
}||z  }t        j                  ||d      }t        j                  ||      }t        j                  ||d      }t        j                  ||      }||z
  |z
  }d }d } d }!|d   r||z  |z  }|d   r0|.t        |      dkD  rt        j                  |
|z  |d      } n|
|z  } |d   r8|6t        |      dkD  rt        j                  |
|d      }!n|
j!                         }!t#        ||j                        t#        | ||j                  nd       t#        |!||j                        fS d       fS )Nc              3   h   K   | ])  }|!|j                  t        j                        n| + y wr*  )r9   r@   rT  r  r:   r;   s     r-   r  z-native_layer_norm_backward.<locals>.<genexpr>  s>      9  = 	
e.E.EF	9s   /2r   ri  r#   r$   TF)r  rP   rE   get_computation_dtyper   rz  rU   rm  r  rk  rj  re  rX   r@   r   r   r  r  )#r  r   r  r   r  r   r  r  input_shape
input_ndimgrad_out_cast
input_castweight_cast	bias_castaxis
inner_dims
outer_dimsinner_dim_indicesouter_dim_indicesrg  r  Mrj  x_hat
grad_x_hatabr  r  r  rL   r  d_weightr  r;   s#                                     @r-   native_layer_norm_backwardr  q  s    ++KJ33EKK@9 E640	95M:{I $$$,--DTU#JUd#J#%#%: (9$$Q'$$Q'	( 	ZAZAKQ!V$(=a1f(E,7NEOOK(3>q>EOOK./t3>q>EOOK./
 	
 HL
 	

 T:>>#34DT:>>#34D!!!$$&E"[0
"
QA		*/6A	:u	%B	2($	/B	5"	BEBJE $G!%H#F1~!8u$1~+1 !A%yy!68I5QH$u,H1~)/ !A%YY}.?GF"((*F 	GU[[)Hf.@flldKF$*:DJJE  AEE r,   c          
          t        | |||||||      }||	|
f}t        |      D ]2  \  }}|	t        ||   |j                         t	        |||   d       4 |S r  )r  r  r   r  r   )r  r   r  r   r  r   r  r  r@  rA  r  r  rY   rg  rK   s                  r-   native_layer_norm_backward_outr    sw     (%)4vt[F d#J&! Q1=jmQWW5Q
14PQ
 r,   c                 &   g }t        t        |            D ]'  }|j                  | j                         |z
  dz
         ) t	        j
                  | j                        }| j                  |      }|~|t        j                  t        j                  fv r.t        j                  t        j                        j                  }n0t        j                  t        j                        j                  }n|}t        j                  t        j                  j                   j"                  j%                  t        j&                  |d      j)                  |d      |            }	|j+                  |	      }
||
j+                  |      }
| j,                  xs |d uxr |j,                  }t	        j.                  |       }|t        j0                  t        j2                  fv }|s"|s |
j5                         }
|	j5                         }	|
j7                  |       }||	fS )Nr#   r$   Tr   )rU   rz  rm  rP   rE   r  r   r9   r@   r   r   finfor6  r   rsqrtopsr   addScalarrR  r   r   	is_nestedr   channels_lastchannels_last_3dr  r+  )r   r  r   r6  dims_to_reducerg  r;   upcasted_inputeps_valrqrst_inputupcasted_resultr  rP  is_channels_lastr  s                  r-   _fused_rms_normr    s    !#N3'() 3eiikAo123 33EKK@XX/0N { @@kk%--044Gkk%--044G++ 			!!IIna(--.$-OQX	
K %((5O)--f5 LF$$6$K6;K;KI//6M$) 
 -)446!,,. $$U+F;r,   c                 :   |j                   }|j                         }t        j                  |j                        }| j                  |t        j                        }	|j                  |t        j                        }
|!|j                  |t        j                        nd }|	J |t        |      z
  }||d  }|d | }g }g }t        |      D ]*  }||k\  r|j                  |       |j                  |       , t        |      }t        |      }ddlm}  ||dk(        s ||dk(        r4|d   r|j                  |      nd |d   r|j                  ||d        fS d fS t        ||
j                               }||	|z  }n|	}d }d }|
|z  }|d   r)t        j                   ||z  |d      }|||z  |z  z
  |z  }|d   r0|.|	|z  }t        |      dkD  rt        j                   ||d      }n|}t#        ||j                        t#        ||j                        fS )NrO  r   rJ  r#   Tr   F)r  rP   rE   r  r   r9   r@   rT  rz  rU   rm  r  rk  rK  re  rX   r   r  )r  r   r  r  r   r  r  r  r;   r  r  r  r  r  r  r  r  rg  r  r  rK  r  r  r  r  sum_vald_weight_full_shapes                              r-   _fused_rms_norm_backwardr	    s[    ++KJ33EKK@KK)@)@   M +5;R;RSJ  			#53J3J	K 
 $$$,--DTU#JUd#J#%#%: (9$$Q'$$Q'	( 	ZAZADa1fQ!7,7NEOOK(3>q>EOOK./
 	
GK
 	

 T:>>#34D"[0
"
 $G!%HE1~))EJ.4EtTg 55=1~+1+e3 !A%yy#):EH +H 	GU[[)Hekk* r,   running_meanrunning_varmomentum
functionalc	                 2   dgt        t        d| j                                     z   }	t        j                  | j
                        }
|}|}|r"t        j                  | j
                        }
| j                  |
      }t        j                  ||	dd      \  }}t        j                  ||z         }| |z
  |z  }t        j                  ||	      }t        j                  ||	      }|!||z  d|z
  |z  z   }|s|j                  |       |2| j                         | j                  d   z  }t        j                  ||	      }|||dz
  z  z  }||z  d|z
  |z  z   }|s|j                  |       n||J |j                  |
d      }|}|j                  |
d      }|}|}dt        j                  ||z         z  }| j                  j                   dk7  r|}|}n"| j#                  d	      }| j#                  d	      }t%        || j                         dz
        }t%        || j                         dz
        }| |z
  |z  }|2|j'                         }t%        || j                         dz
        }||z  }|2|j'                         }t%        || j                         dz
        }||z   }| j                  j                   dk(  r8|j                  | j
                        }|j                  | j
                        }|j                  | j
                        ||||fS )
Nr   r$   r   T)rP   
correctionr   r#   )r   r   r  r   )rm  rU   rP   rE   r  r   r9   r@   var_meanr  r  copy_r   r  rX  r  r  re  rX   r3  )r   r   r  r
  r  r   r  r6  r  reduction_dimsr;   new_running_meannew_running_var	input_acc
biased_varr   r  r  	save_mean	save_rstdnsqueezed_varunbiased_varinvstds                           r-   native_batch_norm_helperr  b  s    S4a 566N33EKK@#!O!77DHH#4H5	 >>>a

D {{:+,$,$&MM$7	MM$7	#')3q8||6SS""#34"A.A !==^DL'1A;7L&5X8TTO!!/2'K,CCC#->TJ'!nn+<4nH%ejjs!234<<%$II-I-I uyy{Q7"6599;?;$,&(!"6599;?;&||~ uyy{Q7$||E!LLu{{L3	LLu{{L3				$ r,   r  save_invstdc                 >    t        | |||||||d	      \  }}	}
}}||	|
fS NFr  r   r   r  r
  r  r   r  r6  r  r  r  rW   s               r-   native_batch_normr$    s=     *Bvt\;(CQV*&FIy!Q 9i''r,   c           
          ||t         j                  | |||||      S |t        d      |t        d      |rt         j                  | |||||||      S t         j                  | ||||||      S )Nz`running_mean is None, but running_var is provided. They should both be None or both be provided.z`running_var is None, but running_mean is provided. They should both be None or both be provided.)r   _native_batch_norm_legitrl  $_native_batch_norm_legit_no_training)r   r   r  r
  r  r   r  r6  s           r-   native_batch_norm_decompositionr(    s      3,,648S
 	
 <
 	
 <
 	
 ,,64{HhPS
 	
 8864{Hc
 	
r,   c                 l   | j                  |      }||z   dz
  |z  }|dk(  rZ|dk(  rU|D cg c]  }| }}|||z  |z
  z
  ||dz
  <   t        j                  j                  j                  j                  | ||      S t        j                  j                  j                  j                  | ||      S c c}w Nr#   r   )r$  r@   r  r   r  r  r  r   )ro  r  rP   r  r  rW   r  s          r-   unsafe_chunk_py_implr+    s    {{3HV#a'F2JQ8q=+12az22",
V0Ch0N"OFQJyy~~55==fkSVWW99>>&&--fj#FF 3s   	B1c           
      N    t         j                  j                  | ||||d||      S r!  )r   r&  r  )r   r   r  r
  r  r  r6  s          r-   r'  r'    s5     ((00	 	r,   c                 >    t        | |||||||d	      \  }}	}
}}||	|
fS r!  r"  r#  s               r-   r&  r&    s=     *Bvt\;(CQV*&FIy!Q 9i''r,   c                 >    t        | ||d d |||d	      \  }}}}	}	|||fS r!  r"  )
r   r   r  r   r  r6  r  r  r  rW   s
             r-   !_native_batch_norm_legit_no_statsr/  (  s<     *BvtT48S%*&FIy!Q 9i''r,   c                 f    t        | |||||||d	      \  }}	}
}}|J d       |J d       ||	|
||fS )NT#new_running_mean should not be None"new_running_var should not be Noner"  )r   r   r  r
  r  r   r  r6  r  r  r  r  r  s                r-   #_native_batch_norm_legit_functionalr3  7  sl    " 	!vt\;(CQU	 'N)NN'&L(LL&9i)9?JJr,   c           	      T   t         j                  j                  | ||||d|      }d}|t         j                  j                  j                  k(  r t         j                  j                  | |      }t        j                  |t         j                  | j                  | j                        S )a  
    Return a reserve tensor for batch norm, used only by cudnn to pass forward state to the
    backward pass. This is needed for `_batch_norm_with_update` and `_batch_norm_no_update`,
    which support a variety of backends including cudnn. We create this tensor here to get
    the correct shape in the traced graph if we detect that will call the cudnn kernel,
    and rely on DCE to avoid materializing this tensor.
    Tr   )r   layoutr  )
r@   _C_select_batch_norm_backend_BatchNormBackendCudnn(_get_cudnn_batch_norm_reserve_space_sizeemptyuint8r5  r  )	r   r   r  r
  r  r6  r   backendreserve_sizes	            r-   _get_batch_norm_reserve_tensorr?  P  s      hh11vt\;cG L%((,,222xxHH8
 ;;EKKU\\ r,   c                 d    t        | ||||d||d	      \  }}}	}
}
t        | |||||d      }|||	|fS )NTFr   r  r?  r   r   r  r
  r  r  r6  r  r  r  rW   reserves               r-   _batch_norm_with_updaterE  m  sa     *B
*&FIy!Q -vt\;dG 9i00r,   c                     t        | ||||d||d	      \  }}}	}
}t        | |||||d      }|
J d       |J d       |||	||
|fS )NTrA  r1  r2  rB  )r   r   r  r
  r  r  r6  r  r  r  new_rmnew_rvrD  s                r-   "_batch_norm_with_update_functionalrI    s      	!vt\;hT	 -vt\;dG DDDCCCIy'66BBr,   c                 d    t        | ||||d||d	      \  }}}	}
}
t        | |||||d      }|||	|fS )NFrA  rB  rC  s               r-   _batch_norm_no_updaterK    sa     *B
*&FIy!Q -vt\;eG 9i00r,   c                     |J t        j                  |       |k  j                  t         j                        }|j	                  |       | z  d|z  z  }||fS )Nr   rh   )r@   rD  r9   r<  r+  )r   rI  	generatorr  rF  s        r-   _fused_dropout_decompositionrN    s[     OOE"Q&***=D
,,u

%q
1C;r,   )r   r5  r  
pin_memorynon_blockingrP  r  rO  rP  rP  c                   |r|t         j                  k(  sJ d       |rJ d       t        | t         j                  t        t
        t        t        f      sJ |0|.|,t        | t         j                        r| j                         S | S d}t        | t         j                        r| }nt        j                  |       }|c||j                  k7  rT|1|j                  dk(  r"t         j                  j                  ||      }d}t         j                  j                  |||      }|$|s"t         j                  j                  ||      }d}|t        j                  ||      S |S )NTODOFr  TrO  )r@   stridedr8   r   r  r   r  complexr  scalar_tensorr  r  _primsconvert_element_type
device_put)	r:   r   r5  r  rO  rP  rP  dtype_convertedx_tensors	            r-   _to_copyr[    s2    5==08&80!6!>a%,,UD'BCCC~%-M,Aa&779HO!U\\"&&q)f7!5||885IH"O<<**8V\J<<44XuE {{8=AAOr,   c                 ,    t         j                  |       S r6   )r   alias)r:   s    r-   nop_decompositionr^    s     ::a=r,   out3exponential_average_factorepsilonc           
         t         j                  | |||||||      \  }}	}
|r%||	|
| j                  dt        j                        fS ||j                  d      |j                  d      | j                  dt        j                        fS )Nr  r   )r   r$  re  r@   r<  )r   r   r  r
  r  r   r`  ra  r  r  r  s              r-   cudnn_batch_normrc    s     $$"	GAq! 1aU[[ABB	EKK0	 r,   c                     t        |      D ]>  \  }}|dk(  s|| j                  k  r| j                  |   |k(  r.| j                  |      } @ | S r]   )r  ru  r  rV   )r:   broadcast_maskr  r  s       r-   _broadcast_batch_norm_backwardrf   	  sO    / "
d19dQVVm0ED!A" Hr,   rD  c                 *    t        | |||||||||	
      S r6   )native_batch_norm_backward)r  r   r   r
  r  r  r  r<  r6  r  rD  s              r-   batch_norm_backwardri  '	  s/     & r,   c
                   & |j                   }
||j                   }n|
}t        j                  |j                         &&fd| ||||||fD        \  }}}}}}}|j                  }|j	                         }|dk\  sJ d       d}t        t        |            ||   z  }|}|}|r||"J ||J |}t        j                  ||z         }dg|z  }||   ||<   g }t        |      D ]  }||k7  s	|j                  |        t        ||      }d|z  }t        j                  ||      }t        j                  |||z
  z  |      }t        ||z  |      }t        t        j                  ||z  ||z        |      } |t        ||      dz  }!nt        ||z  |      }!|r||z
  | z  }"||"z
  |z
  |!z  }#n||!z  }#|	d   r||z  }$nd }$|	d   r|}%nd }%|#j                  |
      t        |$|      t        |%|      fS )Nc              3   H   K   | ]  }||j                        n|  y wr6   r  r  s     r-   r  z-native_batch_norm_backward.<locals>.<genexpr>^	  s,      	 $%=a7	s   "r$   z$rank of the input must be at least 2r#   rh   )r   rE   r  r  rP   r  rm  r@   r  rU   rm  rf  r   r   r9   r  )'r  r   r   r
  r  r  r  r<  r6  r  r  weight_dtyper  r  r  running_mean_castrunning_var_castsave_mean_castsave_invstd_castr  
input_rankr  num_featuresr   r  re  reduction_axesrg  r   grad_output_sumdot_p	grad_mean
proj_scale
grad_scaleprojrY   rd  	grad_biasr;   s'                                         @r-   rh  rh  C	  sx    ++K||"33EKK@	 
	 ++KJ?BBB?D[)*[->>LDFF$666 !,1A1MMM -34!"j 0N&t,N4 "N: %9!!!$% *$?DDii~>OIImzD'89>JE./E~VI/		%$,0J
 3FNKcQ
3[ .

 T!Z/$t+y8JF
"Z/
1~fn1~#		 	k"K.I|, r,   c
                    t        | |||||||||	
      }|
||f}t        |      D ]2  \  }}|	t        ||   |j                         t	        |||   d       4 |S r  )rh  r  r   r  r   )r  r   r   r
  r  r  r  r<  r6  r  r@  rA  r  r  rY   rg  rK   s                    r-   native_batch_norm_backward_outr|  	  s    " (F d#J&! Q1=jmQWW5Q
14PQ
 r,   save_varc                 B    t         j                  || |||||d|g d
      S NT)TTTr   rh  )r   rm   r   r
  r  r  r}  ra  s           r-   miopen_batch_norm_backwardr  	  s5     ** r,   reserveSpacec	                 B    t         j                  || |||||d|g d
      S r  r  )	r   rm   r   r
  r  r  r}  ra  r  s	            r-   cudnn_batch_norm_backwardr  	  s5     ** r,   c                    | j                   | j                  t              t        j                  dv fd       | j                  dd  D ]  }t        j                  |dk7  fd         d   |d   z  dk(  rxd   |d   z  dk(  rjt        d t        dd  |      D              }t        d t        dd  ||      D              }t        j                  j                  j                  | ||      S d	 d
 fd} |d   |d         \  }}}}	 |d   |d         \  }
}}}| dt        |d      |
f   }|	s|st        j                  |d      S d } |||||	d      \  }} |||||d      \  }}d }t        t        |j                  d         t        |j                  d               D ]!  \  }}||d|d d |f   }||d|d d |f   z   }# |||z  z  S )Nr  c                      d  S )Nz9adaptive_avg_pool2d(): Expected 3D or 4D tensor, but got r+   ru  s   r-   r   z%adaptive_avg_pool2d.<locals>.<lambda>
  s    KD6R r,   rQ  r   c                  "    dt                dS )Nzjadaptive_avg_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has shape r  r  r  s   r-   r   z%adaptive_avg_pool2d.<locals>.<lambda>
  s     99>uaI r,   rT   c              3   ,   K   | ]  \  }}||z    y wr6   r+   )r  rg  r&  s      r-   r  z&adaptive_avg_pool2d.<locals>.<genexpr>
  s     G$!QqAvG   c              3   :   K   | ]  \  }}}||d z
  |z  z
    ywr#   Nr+   )r  rg  r&  r  s       r-   r  z&adaptive_avg_pool2d.<locals>.<genexpr>
  s'      
 '1aAQ!O
s   c                 8    t        j                  | |z  |d      S )Ntruncrounding_moder@   divr  r  r  s      r-   start_indexz(adaptive_avg_pool2d.<locals>.start_index"
  s    yyQ99r,   c                 J    t        j                  | dz   |z  |z   dz
  |d      S )Nr#   r  r  r  r  s      r-   	end_indexz&adaptive_avg_pool2d.<locals>.end_index%
  s&    yy!a%1q1,awGGr,   c                    t        j                  |t         j                        } |||       }| |z  dz   }| |z  }|dk(  xs ||z  dk(   }|r|dz  }n
|dk(  r|dz  }t        j                  |t         j                        }|j                  d      |z   }|rUt        j                  | dz
  |j
                  |j                        }	t        j                  ||	      } |||       }
|
|z
  }n|}||||fS )Nr  r#   r   rT   r  )r@   r  r  rV   rU  r   r  minimum)in_sizeout_sizeorangei0	maxlengthin_size_modadaptive	range_maxr  maxvali1lengthr  r  r  s               r-   compute_idxz(adaptive_avg_pool2d.<locals>.compute_idx(
  s   hvU[[I73 x'!+	(#q(GH{,Ba,GHNIANILL6M	ll2* ((!399SZZF --V,C 68W5B"WFFFIx//r,   .r9  )r   rT   r   c                     t        |t              r| |fS |dk  sJ ||j                  d      k\  }|dk(  rt        |d      }t	        j
                  | |d      } t        ||       }| |fS )Nr   rT   rQ  r9  r   )r8   r   rV   rX   r@   ra  )valsr  r  r  rP   r  s         r-   
maybe_maskz'adaptive_avg_pool2d.<locals>.maybe_maskP
  sw    fg&< 7N7 0 0 44Dby(q1$$T45D&vt4F<r,   )r  rP   r   )r  r  rz  r@   r   r  r  nnr  
avg_pool2drX   r   r
   rU   )r   r  r  rn  kernelr  idxhlength_hrange_max_h
adaptive_hidxwlength_wrange_max_w
adaptive_wr  r  retrg  jr  r  ru  r  r  s                      @@@@@r-   adaptive_avg_pool2dr  
  s.   
 \\FKKEu:D	LLR [[ 
FI	

 Ry;r?"a'E"IB,G1,LG#eBCj+*FGG 
+.uRSz;+O
 
 xx""--eVVDD:H0@ /:%)[QS_.U+D(K.9%)[QS_.U+D(K'a0$67Djzz$H--   hjbND(  hjbND(
 Cdjjn-uTZZ^/DE +1;sAq!|$CS!Q\**C	+
 (X%&&r,   c                    t        j                  d| d       t        t        j                  | j
                  d |        }t        t        j                  |      }dg| j                  z  }| j
                  d |  |d |  |t        j                  || j                        j                  |      |z  z   j                  d      }| j                  t        | j
                  d |        t        |      z         }t        j                  |j                  d      |g| j                  d      d      j                  |j
                        S )Nmax_unpoolingd_forward_outr#   r  rT   Fr  )rE   alert_not_deterministicr   operatorr   r  ru  r   r  r  r  r  re  rm  r  )	r|   r  r  rP   nchwindices_nc_shapeindices_flatr  s	            r-   _max_unpoolndr  p
  s    
!!M#m"DE	djj3$/	0B	k	*BsTYY"jj3$/Usd$++b+5::;KLrQQgbk  ^^DEcT!23d;6GGHF!!r\NDLL,< " 
d6<<r,   c                     t        j                  j                  t         j                  k(  fd       t        j                  t	              dk(  fd       t        j                   j
                  dv  fd       t        j                   j                  j                  k(   fd       t        d j
                        D ].  t        j                   j                        dkD   fd	       0 t         d      S )
Nc                  "    d j                    S )Nz2elements in indices should be type int64 but got: r   )r  s   r-   r   zmax_unpool2d.<locals>.<lambda>
  s    DW]]OT r,   r$   c                  "    dt                dS )NzMThere should be exactly two elements (height, width) in output_size, but got 
 elements.rz  r  s   r-   r   zmax_unpool2d.<locals>.<lambda>
      ;'(
4 r,   r  c                  $    d j                    dS )NzLInput to max_unpooling2d should be a 3d or 4d Tensor, but got a tensor with  dimensions.ru  r   s   r-   r   zmax_unpool2d.<locals>.<lambda>
  s    %%)YYK|= r,   c                  <    dj                    d j                    S NzBExpected shape of indices to be same as that of the input tensor (z%) but got indices tensor with shape: r  )r  r|   s   r-   r   zmax_unpool2d.<locals>.<lambda>
  s,    PQUQ[Q[P\ ]229--B r,   r#   r   c                  *    dj                    d  dS )NzZmax_unpooling2d(): Expected input to have non-zero size for non-batch dimensions, but got  with dimension  being empty.r  )rg  r|   s   r-   r   zmax_unpool2d.<locals>.<lambda>
  s%    ::,&6qcH r,   )
r@   r   r   r  rz  ru  r  rU   r$  r  )r|   r  r  rg  s   ```@r-   max_unpool2dr  
  s     
LL$T 
LLKA	
 
LL		V	
 
LL

gmm#	
 1dii  
IIaL1	

 wQ77r,   c                     t        j                  j                  t         j                  k(  d        t        j                   j                  dv  fd       t        j                  t              dk(  fd       t        j                  t              dk(  fd       t        j                  t              dk(  fd       t        j                   j                  j                  k(   fd       t        d	 j                        D ].  t        j                   j                        d
kD   fd       0 t        j                  d
   d
kD  xr d	   d
kD  xr d   d
kD  fd       t         d      S )Nc                       y)Nz(elements in indices should be type int64r+   r+   r,   r-   r   zmax_unpool3d.<locals>.<lambda>
  r  r,   r9  r  c                  $    d j                    dS )NzLInput to max_unpooling3d should be a 4d or 5d Tensor, but got a tensor with r  ru  r   s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s    ^_d_i_i^jjvw r,   r   c                  "    dt                dS )NzVThere should be exactly three elements (depth, height, width) in output_size, but got r  r  r  s   r-   r   zmax_unpool3d.<locals>.<lambda>
  r  r,   c                  "    dt                dS )NzRThere should be exactly three elements (depth, height, width) in stride, but got: r  r  rn  s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s    dehioepdqq{| r,   c                  "    dt                dS )NzSThere should be exactly three elements (depth, height, width) in padding, but got: r  r  )r  s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s    efijqfress}~ r,   c                  <    dj                    d j                    S r  r  )r  r   s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s,    PQVQ\Q\P] ^229--B r,   r#   r   c                  *    dj                    d  dS )NzZmax_unpooling3d(): Expected input to have non-zero size for non-batch dimensions, but got r  r  r  )rg  r   s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s%     ;;-'7s-I r,   r$   c                      d  S )Nz5strides should be greater than zero, but got stride: r+   r  s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s    GxP r,   )
r@   r   r   r  ru  rz  r  rU   r$  r  )r   r  r  rn  r  rg  s   `````@r-   max_unpool3dr  
  sD    
LL$&X 
LL

fw 
LLKA	
 
LLFq| 
LLG~ 
LLw}}$	
 1ejj! 
JJqMA	

 
LLq	A9&)a-9F1IMP
 a88r,   )rn   ro  c                $    t        | |||d|      S )NTinplacern   
_index_addr:   rP   r  ro  rn   s        r-   
index_add_r  
  s     aeVTGGr,   c                $    t        | |||d|      S )NFr  r  r  s        r-   	index_addr  
  s     aeVU%HHr,   r  c                6   t        j                  | j                        t        j                  j                  dk  fd       j                  dk(  rj                  d      nd|j                  dkD  r|j                        ndt        j                  k(  fd       dk7  ret        j                  | j                        t        j                  t        u xs t        j                  t                    fd       |z  }| j                  dk(  }|r| j                  d      n| }dz  fz   }|rt        j                  nt        j                  }	 |	|||d      }
|r| S |r|
j                  d      S |
j!                         S )	Nr#   c                  $    d j                    dS Nz(Index should have dimension 1 or 0 (got r4  ru  r  s   r-   r   z_index_add.<locals>.<lambda>      :5::,aH r,   r   c                      d d d S )NzNumber of indices (z') should be equal to tensor.size(dim) (z), for dim=r+   )rP   
index_sizerp  s   r-   r   z_index_add.<locals>.<lambda>  s!    %j\1XYdXeeqmplrs r,   c                  (    dt                d dS )Nzalpha argument of type z cannot be safely cast to type !)r  )rn   python_types   r-   r   z_index_add.<locals>.<lambda>"  s    -d5k]:YZeYffgh r,   r6   Tr  )rE   canonicalize_dimsru  r@   r   r$  dtype_to_typer   r  is_weakly_lesser_typer  rV   r   
index_put_	index_putr  r  )r:   rP   r  ro  r  rn   zero_dimrL  r  r  r   r  r  rp  s    ``  `     @@@r-   r  r  	  sM    
!
!!&&#
.C	LL

aH #(**/AqJ&,kkAo&++c"1K	LLz!s z))!''24 E**4;Dh	

 %vv{H#QB
C-5(
"C#*I
BV
5C!)s{{1~?s~~/??r,   c           
         t        j                  t        |       dkD  d        t        |       }| d   j                         }|dd  }t	        d | D              }|r||f}n||f}||z   }| d   j                  ||      }dt        |      z  }	t        |      D ]j  }
| |
   }t        j                  ||	d||j                  d      z
  fz   |      }|rt        j                  ||d|
      }Rt        j                  ||d|
      }l |S )Nr   c                       y)Nz#received an empty list of sequencesr+   r+   r,   r-   r   zpad_sequence.<locals>.<lambda>4  r  r,   r#   c              3   >   K   | ]  }|j                  d         ywr  r$  )r  r:   s     r-   r  zpad_sequence.<locals>.<genexpr>8  s     /!&&)/s   )r   r   rP   r  )
r@   r   rz  r$  r   r>  rU   r   rk  r  )	sequencesbatch_firstpadding_valuesequences_sizemax_sizetrailing_dimsmax_lenout_dimsr   dim_paddingsrg  currseqrows                r-   pad_sequencer  1  s#    
LLY!#%RS^N|  "HQRLM/Y//G"G,^,-'H
A,

-
8CC..L>" @A,""\Q',,q/(A$BBM
 %%c3AQ%?C%%c3AQ%?C@ Jr,   c                 "    t        | |||d      S )NTr  _index_copyr:   rP   r  ro  s       r-   index_copy_r
  L  s    q#ufd;;r,   c                 "    t        | |||d      S )NFr  r  r	  s       r-   
index_copyr  Q  s     q#ufe<<r,   c                   t        j                  | j                  |      }t        j                  j                  dk  fd       | j                  dk(  }|r| j                  d      n| }j                  dk(  rj                  d      nd|z  fz   }|rt        j                  nt        j                  } ||||      }	|r| S |r|	j                  d      S |	j                         S )Nr#   c                  $    d j                    dS r  ru  r  s   r-   r   z_index_copy.<locals>.<lambda>]  r  r,   r   r6   )rE   r  ru  r@   r   rV   r   r  r  r  r  )
r:   rP   r  ro  r  r  rL  r  r  r   s
     `       r-   r  r  W  s     
!
!!&&#
.C	LL

aH
 vv{H#QB"'**/EOOAuE
C-5(
"C#*I
BV
$C!)s{{1~?s~~/??r,   c                 *   t        j                  | j                  d      |       }t        j                  t        j                  |              }| j
                  s| j                  r| j                  d      }n|}|t        j                  |      z
  |fS )Nr+   r  )r@   r  re  ri   r   r  is_xpur=  )r|   r   rk   r   s       r-   log_sigmoid_forwardr  m  sn     --r*D
1C		599T?"#A||t{{%Q''r,   lowhighrM  c                     t        j                  | j                  t        |      t        |      | j                  | j
                  |      S )N)r  r  r   r  rM  )prims_uniform_helperr  r   r   r  )r:   r  r  rM  s       r-   uniformr  z  s=       	cNt_ggxx r,   c                 <    | j                  t        | |||            S r6   )r  r  )r|   r  r  rM  s       r-   uniform_r    s    ::gdCy9::r,   c                 "   t        |       dz
  }|>t        j                  |d u d        t        j                  t        |      |k(  d        |S |t        j                  |d u d        t        j                  t        |      |k(  d        g }t        |      D ]Z  \  }}t	        |      |k(  r$|j                  | |dz      t	        |      z         8|j                  t        | |dz      |z               \ |S t        j                  dd        y )Nr$   c                       yNz9Must specify exactly one of output_size and scale_factorsr+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>  r  r,   c                       yN r+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>  r  r,   c                       yr  r+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>  r  r,   c                       yr  r+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>  r  r,   Fc                       yr  r+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>  r  r,   )rz  r@   r   r  r  rm  r   )r-  r  scale_factorsspatial_dimensionsrg  r  s         r-   upsample_compute_output_sizer%    s   Z1,T!O	
 	S%);;ZH 4O	
 	S'+==zJm, 	CDAq1v{"":a!e#4s1v#=>""7:a!e+<q+@#AB		C
 	LLRr,   c                     | y | |   S r6   r+   )scalesr  s     r-   get_scale_valuer(    s    ~#;r,   r#  c                 x    t        | j                         ||      }|r|nd gt        |      z  }t        | ||      S r6   r%  r$  rz  _upsample_nearestr   r  r#  osizer'  s        r-   _upsample_nearest_vecr.    s@     ){MRE&TFSZ,?  UE622r,   c                 |    t        | j                         ||      }|r|nd gt        |      z  }t        | ||d      S NTexactr*  r,  s        r-   _upsample_nearest_exact_vecr3    sB     ){MRE&TFSZ,?  UE6>>r,   c                    g }t        |      }|rdnd}t        |      D ]  }||   }| j                  | |z      }	||   |	|	||   z  z  n|	|z  }
t        j                  |t        j
                  | j                        }||z   |
z  j                  t        j                        }t        |dz
  |z
        D ]  }|j                  d      } |j                  |        |S )Nr   r   r  r#   rT   )rz  rU   r  r@   r  r   r  r9   r  rV   rm  )r   r  r'  r2  r  num_spatial_dimsr  r  r-  isizero   output_indicesinput_indicesrW   s                 r-   !_compute_upsample_nearest_indicesr9    s     G;'SsF#$ & A--12/5ay/D*+%RW-e5==V(61U:>>u{{K'!+a/0 	8A)33B7M	8}%+&, Nr,   )preserve_memory_formatr  r'  c                     t        | ||g      S r6   r+  r   r  r'  s      r-   upsample_nearest1dr>    s     UK&::r,   c                 "    t        | ||gd      S r0  r<  r=  s      r-   upsample_nearest_exact1dr@  
  s     UK&FFr,   scales_hscales_wc                      t        | |||g      S r6   r<  r   r  rA  rB  s       r-   upsample_nearest2drE    s     UK(H1EFFr,   c                 $    t        | |||gd      S r0  r<  rD  s       r-   _upsample_nearest_exact2drG  %  s     UK(H1ETRRr,   scales_dc                 "    t        | ||||g      S r6   r<  r   r  rH  rA  rB  s        r-   upsample_nearest3drK  4  s     UK(Hh1OPPr,   c                 &    t        | ||||gd      S r0  r<  rJ  s        r-   _upsample_nearest_exact3drM  B  s!     {Xx:$ r,   r2  c                 D   t        | |||      }d d g|z   }t        j                  | |      }|j                  dk(  rdt	        j
                  |       }| j                  d   }| j                  j                  dk(  r|dk  rt        j                  }|j                  |      }|S )Nr1  r9  r#   cudarO  )r9  r   _unsafe_indexru  rE   r   r  r  r  r@   rT  r  )	r   r  r'  r2  spatial_indicesr  r  rP  
n_channelss	            r-   r+  r+  T  s     8{F%O Tl_,Gw/F{{a33E: [[^
<<&:>!33M"""?Mr,   c           	          |r|rd}n|rd}n|rd}nd}t        |       |z  dk(  sJ t        |              t        dt        |       |      D cg c]  }t        | |||z           c}S c c}w )Nr  r9  r   r$   r   )rz  rU   r  )params
has_biaseshas_projections
group_sizerg  s        r-   gather_paramsrX  o  s{    o
	
	

v;#q(5#f+5(38CK3T./fQZ()  s   A(c                 ~    |r'| d|z     |d|z     }}| d|z  dz      |d|z  dz      }}n| |   ||   }}d\  }}||||fS )Nr$   r#   NNr+   )rT  hiddensrg  bidirectional
cur_params
cur_hiddenbidir_paramsbidir_hiddens           r-   params_hiddensra    sk    !'AAJ
%+AEAI%6A	8Jl!'GAJJ
%/"lz<==r,   c                     ||kD  sJ |j                  | j                  d|||z
               | j                  dd|      S rt   )rm  r%  )r^  last_batch_size
batch_sizer[  s       r-   update_hidden_for_packedre    sE    Z'''NN:$$Q
Oj4PQRQ:..r,   c           	      t    ||k(  r| S ||k  sJ t        j                  | |j                  d|||z
        f      S rt   )r@   concatr%  )r^  rc  rd  
inp_hiddens       r-    update_hidden_for_packed_reverseri    sP     *$Z'''<<a*2NO	
 r,   c           	      X   |d   }|d   }|r|d   nd }	|r|d   nd }
g }g }|r|d   n|d   }|j                  dd|      }t        j                  | t        |            }|r|d d d   }|D ]V  } | j                  d   }||k(  rn|rt        ||||      }nt        ||||      } || |||	||
      }|}|j                  |       X |r|j                          n!|j                  |       |j                          t        j                  |d      }|st        j                  |d      n|}||fS )Nr   r#   r$   r   rT   )
r%  r@   r  rm  r  ri  re  rm  reverser&  )inphiddenrT  rU  	hidden_fnbatch_sizesrk  	ih_weight	hh_weightih_biashh_biasstep_outputr[  rc  r^  	split_inprg  r   
hidden_outs                      r-   one_layer_rnn_datarw    sR    q	Iq	I%fQi4G%fQi4GK"$G)0k"ok!nOq!_5JCk!23IddO	 'IIaLa9OQJ 2OQJ sJ	7IwW
:&#'& z"
))K
#C.57A&:J
?r,   c                       fd}|S )Nc                 D     t        j                  |||      | z         S r6   r   linearrg  r^  rp  rr  rq  rs  nonlinearitys         r-   rL   zrnn_cell.<locals>.inner  s     AHHZGDqHIIr,   r+   r}  rL   s   ` r-   rnn_cellr    s    J Lr,   c                       fd}|S )Nc                 r    t        j                  | ||      }  t        j                  |||      | z         S r6   rz  r|  s         r-   rL   zrnn_cell_data.<locals>.inner  s2    HHQ	7+AHHZGDqHIIr,   r+   r~  s   ` r-   rnn_cell_datar    s    J Lr,   c           	      x   |d   }|d   }|r|d   nd }|r|d   nd }	t        j                  | ||      }
|r|
j                  d      n|
}
|j                  d      }g }|
D ]   } |||||||	      }|j	                  |       " |r|j                          t        j                  |d      }||j                  d      fS )Nr   r#   r$   r   )	r   r{  fliprV   rm  rk  r@   r&  r  )rl  rm  rT  rU  rn  rk  rp  rq  rr  rs  precomputed_inputr^  rt  rg  r   s                  r-   one_layer_rnnr    s    q	Iq	I%fQi4G%fQi4Gi95<)..q1BS!!!$JK 'q*i)WU
:&' 
))K
#C
""1%%%r,   c                    |d   }|d   }|r|d   }|d   }nFt        j                  |j                               }t        j                  |j                               }|d   j                  d      }	|d   j                  d      }
g }d}|	j                  d      }d}d}d}d}| j	                         } |	j	                         }	|
j	                         }
t         j
                  j                  j                  j                  | |||||	|
|||||||||      }|d   |d   |d   }}}||j                  d      |j                  d      ffS )Nr   r#   r$   r   F)
r@   r  r$  rV   r  r  r   mkldnn_rnn_layerr  r  )rl  rm  rT  rU  rk  w0w1w2w3hxcxro  modehidden_size
num_layersr\  r  r<  outputsr[   hycys                         r-   mkldnn_one_layer_lstmr    sS   	B	BAYAY[[#[[#			Q	B			Q	BKD''!*KJ MKE ..
C	B	Biinn--55





!G$ 
GAJ
2rArzz!}bjjm,,,r,   c
                    |r| j                  dd      n| } g }
t        |      D ]  }t        ||||      \  }}}}|r
||dz
  k  r|nd} |	| |||      \  }}|
j                  |       |r! |	| |||d      \  }}|
j                  |       |r*t	        j
                  |g|j                         dz
        } n|} |dk7  s|s||dz
  k  st	        j                  | |d      }  |r| j                  dd      n| } | |
fS )Nr   r#   r   T)rk  )r<  )	transposerU   ra  rm  r@   r&  rP   r?  )r   rm  rT  rU  r  r?  r<  r\  r  layer_fnfinal_hiddensrg  r]  r^  r_  r`  fwd_inp
fwd_hiddenbwd_inp
bwd_hiddens                       r-   _rnn_helperr  #  s"    &1EOOAq!eEM: >=KFA}>
:
Jl $
QU(:'&uj*jQZ("*|\:t#GZ   ,IIw0'++-!2CDEEa<Ea*q.&8MM%=E)>, &1EOOAq!eE-r,   c	                     |j                  d      }	t        ||d      }t        | |	|||||||t        t        t        t        j                              
      \  }
}|
t        j                  |d      fS Nr   Frn  )	unbindrX  r  r   r  r  r@   r   stackr   r  rT  rU  r  r?  r<  r\  r  rm  r   r  s               r-   rnn_tanh_inputr  L  t     YYq\F6:u5F$%**)=>C M1---r,   c	                     |j                  d      }	t        ||d      }t        | |	|||||||t        t        t        t        j                              
      \  }
}|
t        j                  |d      fS r  )	r  rX  r  r   r  r  r@   r  r  r  s               r-   rnn_relu_inputr  k  r  r,   c	                     |j                  d      }	t        ||d      }t        | |	||||||dt        t        |t        t        j                              
      \  }
}|
t        j                  |d      fS Nr   Fro  rn  )	r  rX  r  r   rw  r  r@   r  r  dataro  r  rT  rU  r  r?  r<  r\  rm  r   r  s               r-   rnn_relu_datar    {     YYq\F6:u5F$##EJJ/	
C  M1---r,   c	                     |j                  d      }	t        ||d      }t        | |	||||||dt        t        |t        t        j                              
      \  }
}|
t        j                  |d      fS r  )	r  rX  r  r   rw  r  r@   r   r  r  s               r-   rnn_tanh_datar    r  r,   c                 l   t        j                  |||      | z   }|j                  d|      }|d   j                         }	|d   j                         }
|d   j	                         }|d   j                         }|
|z  |	|z  z   }||j	                         z  }||nt        j                  ||d       }||fS )Nr9  r   r#   r$   r   r   r{  chunkr   r   )rl  r  r  rq  rs  	hr_weight	chunk_dimgateschunked_gatesin_gateforget_gate	cell_gateout_gater  r  s                  r-   	lstm_cellr    s    HHRG,s2EKK9-MA&&(G"**,Ka %%'IQ'')H	r	Wy0	1B	BGGI	B ahhr9d&CBr6Mr,   c           
      (   |d   }|d   }|r|d   nd }|r|d   nd }t        |      dk(  r|d   nt        |      dk(  r|d   nd }	|d   j                  d      }
|d   j                  d      }t        j                  | ||      }|r|j	                  d      n|}g }|D ](  } t        | |
||||	d      \  }
}|j                  |
       * |r|j                          t        j                  |d      }||
j                  d      |j                  d      ffS )Nr   r#   r$   r   r  r9  r  )rz  rV   r   r{  r  r  rm  rk  r@   r&  r  )rl  rm  rT  rU  rk  rp  rq  rr  rs  r  r  r  r  rt  r   s                  r-   one_layer_lstmr    s*   q	Iq	I%fQi4G%fQi4G[A%q	Fq8H6!9d  
		Q	B			Q	Bi95<)..q1BSK  3B	7IQRSB2 
))K
#CA

1...r,   c           
         |d   }|d   }|r|d   nd }|r|d   nd }	t        |      dk(  r|d   nt        |      dk(  r|d   nd }
g }g }|r|d   n|d   }t        j                  | t        |            }|r|d d d   }|d   }|d   }|j	                  dd|      |j	                  dd|      }}|D ]  } | j
                  d   }t        j                  | ||      } ||k  ra|j                  |j	                  d|||z
        |j	                  d|||z
        f       |j	                  dd|      |j	                  dd|      }}||kD  rXt        j                  ||j	                  d|||z
        fd      }t        j                  ||j	                  d|||z
        fd      }t        | ||||	|
d      \  }}|}|j                  |        |r|j                          ||f}nZ|j                  ||f       |j                          t        | \  }}t        j                  |d      t        j                  |d      f}t        j                  |d      }||fS )	Nr   r#   r$   r   r  r9  rT   r  )rz  r@   r  rm  r%  r  r   r{  rm  rg  r  rk  r  r&  )rl  rm  rT  rU  ro  rk  rp  rq  rr  rs  r  rt  r[  rc  ru  orig_hxorig_cxr  r  rg  rv  hidden0hidden1r   s                           r-   one_layer_lstm_datar    sy   q	Iq	I%fQi4G%fQi4G[A%q	Fq8H6!9d  KG)0k"ok!nOCk!23IddO	QiGQiGq!_-q!_- 	B
  IIaLhhsIw/ NNIIaOa$78IIaOa$78 YYq!Q'1a);B W^^AO8KLMqB W^^AO8KLMqB 3B	7IQRSB236 "X
Bx =YYw*EIIgq,AA

))K
#C
?r,   c                 4    d } || ||      rt         S t        S )a*  Check whether we could use decompose lstm with mkldnn_rnn_layer.
    All the below conditions need to be met:
        * ``torch._C._get_mkldnn_enabled()`` returns ``True``.
        * All the input args are on CPU.
        * The dtypes of args are either torch.float or torch.bfloat16.
        * Inference.
        * ``has_projections`` returns ``False``.

    Args:
        * input: the input sequence to LSTM
        * hx: a tuple of the input hidden state and cell state ``(h_0, c_0)`` to LSTM
        * params: the weight and bias tensors of LSTM
    c                 N   t         j                  j                         sy| gt        |      z   t        t	        j
                  |            z   }|D ch c]  }|j                   }}t        |      dk7  ry|j                         }|t        j                  d      k7  ry|D ch c]  }|j                   }}|D ]&  }|t         j                  t         j                  fvs& y | j                  ry|d   j                  d      |d   j                  d      k7  }	|	ryyc c}w c c}w )NFr#   r  r   r$   T)r@   r6  _get_mkldnn_enabledrm  r	   from_iterabler  rz  popr   r   bfloat16requires_gradr$  )
r   r  rT  rh  tdevicesr  dtypesr   rV  s
             r-   
use_mkldnnz2select_one_layer_lstm_function.<locals>.use_mkldnnH  s    xx++-'DH$tE,?,?,G'HH%,-188--w<1U\\%((#*+a!''++ 	EU[[%..99	 Q%**Q-2a5::a=8) . ,s   D#D")r  r  )r   r  rT  r  s       r-   select_one_layer_lstm_functionr  9  s!    : %V$$$r,   c	                    t        |      dk(  sJ d       t        |||d   j                  d      |d   j                  d      k7        }t        t	        |d   |d               }	t        | ||      }
t        | |	||||||||

      \  }}t        t	        |       }|t        j                  |d   d      t        j                  |d   d      fS )Nr$   lstm expects two hidden statesr   r#   )	rz  rX  r$  rm  r  r  r  r@   r  )r   r  rT  rU  r  r?  r<  r\  r  rm  r  r   r  s                r-   	lstm_implr  k  s     r7a<999<6:r!uzz!}1

1/MNF#beRU#$F-eR@H$C m,-MM!,a0%++mA>NPQ2RRRr,   c	                    t        |      dk(  sJ d       t        |||d   j                  d      |d   j                  d      k7        }t        t	        |d   |d               }	t        | |	||||||dt        t        |      
      \  }
}t        t	        |       }|
t        j                  |d   d      t        j                  |d   d      fS )Nr$   r  r   r#   F)ro  )
rz  rX  r$  rm  r  r  r   r  r@   r  r  s               r-   lstm_data_implr    s     r7a<999<6:r!uzz!}1

1/MNF#beRU#$F$#=C m,-MM!,a0%++mA>NPQ2RRRr,   c                 &   | j                  dd      }t        j                  |||      j                  dd      }|d   |d   z   j                         }|d   |d   z   j                         }	|d   |d   |z  z   j	                         }
||
z
  |	z  |
z   S )Nr   r#   r$   r   )r  r   r{  r   r   rl  r^  rp  rr  rq  rs  chunked_igateschunked_hgates
reset_gate
input_gatenew_gates              r-   gru_cellr    s    YYq!_NXXj)W=CCAqIN #nQ&77@@BJ #nQ&77@@BJq!^A%6%CDJJLH!Z/(::r,   c                 P   t        j                  | ||      j                  dd      }t        j                  |||      j                  dd      }|d   |d   z   j                         }|d   |d   z   j                         }	|d   |d   |z  z   j	                         }
||
z
  |	z  |
z   S )Nr   r#   r   r$   r  r  s              r-   gru_cell_datar    s    XXc9g6<<QBNXXj)W=CCAqIN #nQ&77@@BJ #nQ&77@@BJq!^A%6%CDJJLH!Z/(::r,   c	                     t        ||d      }t        | |j                  d      ||||||dt        t        |t
              
      \  }	}
|	t        j                  |
d      fS )NFr   r  )rX  r  r  r   rw  r  r@   r  )r  ro  r  rT  rU  r  r?  r<  r\  r   r  s              r-   gru_impl_datar    si     6:u5F$
		!"}UC M1---r,   c	                     t        ||d      }t        | |j                  d      |||||||t        t        t
              
      \  }	}
|	t        j                  |
d      fS )NFr   r  )rX  r  r  r   r  r  r@   r  )r   r  rT  rU  r  r?  r<  r\  r  r   r  s              r-   gru_implr    sf     6:u5F$
		!2C M1---r,   c                     t        | j                         ||      }t        |d      }t        |d      }t        j                  j
                  j                  | ||||      S Nr   r#   )r%  r$  r(  r@   r  r   _upsample_bilinear2d_aar   r  align_cornersr#  r-  scale_hscale_ws          r-   upsample_bilinear2d_aa_vecr    sV     ){MREmQ/GmQ/G99>>11umWg r,   c                     t        | j                         ||      }t        |d      }t        |d      }t        j                  j
                  j                  | ||||      S r  )r%  r$  r(  r@   r  r   _upsample_bicubic2d_aar  s          r-   upsample_bicubic2d_aa_vecr    sV     ){MREmQ/GmQ/G99>>00umWg r,   c                 z    t        | j                         ||      }|r|nd gt        |      z  }t        | |||      S r6   )r%  r$  rz  _upsample_linear)r   r  r  r#  r-  r'  s         r-   _upsample_linear_vecr    s=     ){MRE+]$#e*1DFE5-@@r,   r  c                      t        | |||g      S r6   r  )r   r  r  rB  s       r-   upsample_linear1dr  "  s     E;zJJr,   c                 "    t        | ||||g      S r6   r  )r   r  r  rA  rB  s        r-   upsample_bilinear2dr  -  s     E;(?STTr,   c                 $    t        | |||||g      S r6   r  )r   r  r  rH  rA  rB  s         r-   upsample_trilinear3dr  <  s!     {MHh+I r,   c                 L    |r|dkD  r| dz
  |dz
  z  S dS |
|dkD  rd|z  S | |z  S )Nr#   rh   r   r+   )r  r  r  ro   s       r-   _compute_scaler  M  sB    5=\#(S.1HqH#/EAIsU{U7XCUUr,   c                 &    |r| |z  S | |dz   z  dz
  S Nr   r+   )ro   	dst_indexr  s      r-   _compute_source_indexr   T  s$    y  	C(3..r,   weightsweights_precisionc                     t        d t        | |      D              d|dz
  z  z   }||z	  }t        j                  |dd      j	                  t        j
                        S )Nc              3      K   | ]F  \  }}|j                  t        j                        |j                  t        j                        z   H y wr6   )r9   r@   r1  )r  r  r  s      r-   r  z%_sum_tensors_uint8.<locals>.<genexpr>^  s8      26!QU[[ADD--s   AAr#   r      )_sum_tensorsr  r@   r   r9   r<  )r  r  r  r  s       r-   _sum_tensors_uint8r  [  sd      :=c7:K 	
 1$	%'F ((F;;vq#&))%++66r,   c                     t        j                  |       j                         }d}t        j                  ||j                        }d|d|dz   z  z  z   }|dk\  }||j                         z
  S )N   r  r   r#   i   )r@   r  r   r  r  r   )r  
max_weightmax_weight_precision
precisionsvaluesr  s         r-   _compute_weight_precisionr  e  si    W%))+J2:;L;LMJ:zA~!677FgD$((*,,r,   c                      j                   d   } j                   dd  }t        |      }t        j                   t        j                  j
                        \  } fd}t        t        |||            D 	
cg c]  \  }	\  }
}} ||
|||dz
  |	z
         }}}
}	}t        t        |       \  }}}g }t        ddgg|z   D ]c  }d d gt        |      D cg c]  }||   dk(  r||   n||    c}z   }t        j                   |      }t        |      }|j                  |       e t        t        |            D ]p  }	||	   ||	   z
  j!                  dd      j#                        }t        |d d d   |dd d         D cg c]!  \  }}|t%        j&                  ||z
  |      z   # }}}r t        |      dk(  sJ |d   }t        j(                         } j*                  j,                  dk(  r|d	k  rt$        j.                  }t1        |t$        j2                        sJ |j5                  |
      } j7                         s|j9                         }|S c c}}}
}	w c c}w c c}}w )Nr#   r$   rL  c                 |   t        | |	|      }t        j                  |j                        j	                  
      }t        ||	      j                  d      } |j                  |j                  d   gdg|z   }|j	                  t        j                        }|dz   j                  | dz
        }|||fS )Nr  r   r   r   r   r#   r   )
r  r@   r  r  r9   r   r   r  r  r  )inp_sizer  r'  nsqueezescale_factorrg  x_f32r:   xp1r  r   r   s            r-   
get_valuesz$_upsample_linear.<locals>.get_values  s    %h-P LL%,,7:::G%lA}EKKPSKTekk!n@sh/?@HHU[[!1umm1m-a}r,   r   r   rh   rO     rO  )r  rz  rE   rF   rN  INT_TO_FLOATr  r  rm  r
   rU   r   rP  r   rm  reversedr   r9   r@   r   r   r  r  rT  r8   r   r  rC  round)r   r  r  r'  rR  	inp_sizesn_dimsrW   r  rg  r  r  r  xs_f32xsxp1svsr  kr  vxscalev1v2r  rP  r   s   ` `                       @r-   r  r  n  s|    QJABI^F''!AANNHAu
 09	;/0
 +A+(F 	8XvvzA~>F  CL)FB	B1vh'( TluV}U!qtqybed1g=UUuc*#Au-
		!	 eFm$ 
)be#**3477> b1gr!$Q$x0
 B 27F++
 

 r7a<<UF //6M ||F"zB//fell+++];F""$MQ V
s   I
I'&I,r  r  c                 4    | j                   |j                   k(  S r6   r  )r  r  s     r-   is_same_sizer'    s    77aggr,   c                 .    t         j                  | |      S r6   )r   r  )r:   r  rG   s      r-   _reshape_aliasr)    s     99Qr,   c                 .    t         j                  | |      S r6   )r   r  )r:   r  s     r-   rP  rP    s    ::a!!r,   c                 2    t         j                  | |||      S r6   )r   r  )r:   r  r}   r  s       r-   r  r    s    >>!WeZ88r,   c                    |D ]F  }|t        j                  |j                  t         j                  t         j                  fv d        H t        j                  |j                  t         j
                  k(  d        ddlm}  || j                         dk(        r<t         j                  j                  | |      }| j                  |j                  |      S t        t        |            D ]2  }||   }||j                  d| j!                  |      dz
        ||<   4 t"        j%                  | |      j'                  | |      S )Nc                       yNz3tensors used as indices must be long or int tensorsr+   r+   r,   r-   r   z&_unsafe_masked_index.<locals>.<lambda>  r  r,   c                       yNz*tensors used as masks must be bool tensorsr+   r+   r,   r-   r   z&_unsafe_masked_index.<locals>.<lambda>  r  r,   r   rJ  r#   r~  )r@   r   r   r`  r  r  rk  rK  r   _meta_registrationsmeta_index_Tensorr>  r  rU   rz  r   r$  r   rP  ra  )r:   r  r  fillr  rK  meta_resultrg  s           r-   r  r    s    LL

EII66M 
LL

ejj <
 Eaggi1n%//AA!WMzz+++T223w<  ?
q	A>GAJ?
 a)55teTBBr,   c                 L   |D ]F  }|t        j                  |j                  t         j                  t         j                  fv d        H t        j                  |j                  t         j
                  k(  d        | j                         dk(  r| j                         S t        t        |            D ]B  }||   }||j                  | j                  |       | j                  |      dz
        ||<   D |j                  | d      }t        j                  | ||d      S )Nc                       yr.  r+   r+   r,   r-   r   z5_unsafe_masked_index_put_accumulate.<locals>.<lambda>  r  r,   c                       yr0  r+   r+   r,   r-   r   z5_unsafe_masked_index_put_accumulate.<locals>.<lambda>  r  r,   r   r#   r~  Tr  )r@   r   r   r`  r  r  r   r  rU   rz  r   r$  ra  r   r  )r:   r  r  r  r  rg  masked_values          r-   #_unsafe_masked_index_put_accumulater9    s     LL

EII66M 
LL

ejj <
 	wwyA~wwy3w<  H
!&&)QGGAJH
 %%teQ/L!!!Wlt!LLr,   c                    | j                         }d}|dk  rd}|6|dkD  r*dg|z  }|j                  d   ||<   |j                  |      }n|}| |z  } t        j                  ||k7  |d      }	|	j                  |      }
t        j                  | ||
      j                  |       }t        j                  ||k7  |d      }|t        j                  j                  k(  r|dkD  r| j                  dd      }||fS |lj                  | j                        }t        j                  |||
      j                  |      }t        j                  ||k7  |d      }|j                         }n"||k7  j                         j                  |       }|t        j                  j                  k(  r|j                         }||fS |t        j                   j                  k(  r|j                         |z  }||fS )Nr#   r$   r   r+   r   )rP   r  r  r@   rj   rV   gatherr  r"   r(   r}   r>  r  r   r9   r*   r)   )r|   r   r   r   r  r  r  r  wr  safe_target_r  r  wsums                 r-   _nll_loss_forwardr?    s    XXZFKzA:E "(aE+E"AAax++f4fa@K((5L ll4l;CCKPPF[[</;FINN(((VaZ}}R-|##HHTZZ ||A{L9AA+N{{6\14;xxz,.33588>IMM''' < 
inn**	*,<r,   c                     | j                         dkD  r| j                         dk  sJ d       |j                         dk  sJ d       | j                         dk(  xr |j                         dk(  }|sA| j                  d   |j                  d   k(  s"J d| j                   d|j                   d       | j                  d	   }|=|j                         dk(  r|j                         |k(  sJ d
| d|j                          t        | ||||      S )Nr   r$   r0  r#   r1  r2  r3  r4  rT   z/weight tensor should be defined either for all z7 classes or no classes but got weight tensor of shape: )rP   r  r   r?  )r|   r   r   r   r  r6  	n_classess          r-   nll_loss_forwardrB  9  s    88:>dhhjAoP/PP-::<1 E 88:?8vzz|q'8LDJJqMV\\!_< 
$TZZL
6<<.J= 

2I>fjjla/FLLNi4O 
9) E++1<<.	:P
 T669lKKr,   c                      t        | ||||      S r6   )r?  )r|   r   r   r   r  s        r-   nll_loss2d_forwardrD  V  s     T669lKKr,   Ac                 0    |dz   | z  |dz   z
  | z  | z  dz   S )Nr$   r   r#   r+   r:   rE  s     r-   _upsample_cubic_convolution1rH  d  s(    UaK1q5!Q&*Q..r,   c                 <    || z  d|z  z
  | z  d|z  z   | z  d|z  z
  S )Nr     r9  r+   rG  s     r-   _upsample_cubic_convolution2rK  h  s0    UQU]a!a%'1,q1u44r,   r  c                    d}| j                   t        j                   d      k(  rt        j                  | d| z
  gd      }t        j                  | dz   d| z
  gd      }t        ||      }t	        ||      }t        j
                  |d      \  }}t        j
                  |d      \  }}	|||	|fS t        | dz   |      t	        | |      t	        d| z
  |      t        d| z
  |      fS )Ng      r  rh   r   r   r   )r  r@   r  rK  rH  r  )
r  rE  tt1tt2w03w12r  r  r  r  s
             r-    _upsample_get_cubic_coefficientsrQ  l  s    Axx5<<&&kk1cAg,A.kk1s7C!G,!4*32*32cq)Bcq)B2r2~ )S!4(A.(q!4(q!4	
 	
r,   coeffstsc                 P    t        |      }t        d t        | |      D              S )Nc              3   ,   K   | ]  \  }}||z    y wr6   r+   r  r  r  s      r-   r  z+_upsample_cubic_interp1d.<locals>.<genexpr>  s     EHRREr  )rQ  r  r  )rR  rS  coeffs2s      r-   _upsample_cubic_interp1drX    s$    .r2GEFG0DEEEr,   c                 6    t        t        j                  |       S r6   )r   r@   r  )rS  s    r-   r  r    s    %))R  r,   	num_stepsc                     | dk  rt        j                  d||      S |s| dz
  | z  nd}t        j                  | || ||      S )Nr#   r   r  )stepsr  r   )r@   ro  linspace)rZ  r  r   r  r  s        r-   _linspace_from_neg_oner^    sI     A~||AfE::-:)a-9	$A>>1"ayuMMr,   thetahr<  c                    | j                   }| j                  }t        ||||      j                  d|d      }t        ||||      j                  |dd      }t	        j
                  d||      }t        j                  j                  j                  |ddd      }t        j                  j                  j                  |ddd      }t        j                  j                  j                  |d	dd      }||z   |z   S )
Nr#   )r#   r#   r#   r  )r   r$   constantr   r  r  r}   r#   r#   )r$   r   	r   r  r^  r  r@   r  r  r  r  )	r_  r`  r<  r  r   r  grid_xgrid_ygrid_ones	            r-   _make_base_grid_4dri    s    KKE\\F $A}eVDII!QPQRF#A}eVDII!QPQRFzz)5@H XX  $$VjPQ$RFXX  $$VjPQ$RFxx""&&xV*TU&VHF?X%%r,   r  c                    | j                   }| j                  }t        ||||      j                  dd|d      }t        ||||      j                  d|dd      }t        ||||      j                  |ddd      }	t	        j
                  d||      }
t        j                  j                  j                  |ddd      }t        j                  j                  j                  |ddd      }t        j                  j                  j                  |	d	dd      }	t        j                  j                  j                  |
d
dd      }
||z   |	z   |
z   S )Nr#   )r#   r#   r#   r#   r  )r   r   rb  r   rc  )r#   r$   )r$   r#   )r   r   re  )r_  r  r`  r<  r  r   r  rf  rg  grid_zrh  s              r-   _make_base_grid_5drl    s5   KKE\\F#A}eVDII!QPQSTUF#A}eVDII!QPQSTUF#A}eVDII!QPQSTUFzz,eFCH XX  $$VjPQ$RFXX  $$VjPQ$RFXX  $$VjPQ$RFxx""&&xV*TU&VHF?V#h..r,   c                     |\  }}}}t        | |||      }|j                  ddd      | j                  j                  d      z  j	                  d      }|j                  |||d      S )Nr  rT   r   r#   rQ  r$   )ri  r  rV  rV   r   )	r_  r$  r  r  rW   r`  r<  	base_gridgrids	            r-   _affine_grid_generator_4drq    sg    JAq!Q"5!QmLI NN2q!$uxx'9'9!'<<AA"ED99Q1a  r,   c                     |\  }}}}}t        | ||||      }|j                  ddd      | j                  j                  d      z  j	                  d      }	|	j                  ||||d      S )Nrn  rT   r9  r#   rQ  r   )rl  r  rV  rV   r   )
r_  r$  r  r  rW   r  r`  r<  ro  rp  s
             r-   _affine_grid_generator_5drs    sm    MAq!Q"5!QOI NN2q!$uxx'9'9!'<<AA"ED99Q1a##r,   c                     t        j                  t        |      dv d        t        |      dk(  rt        | ||      S t	        | ||      S )Nr  c                       y)NzCaffine_grid_generator needs 4d (spatial) or 5d (volumetric) inputs.r+   r+   r,   r-   r   z'affine_grid_generator.<locals>.<lambda>  r  r,   r9  rn  )r@   r   rz  rq  rs  )r_  r$  r  s      r-   affine_grid_generatorrv    sJ     
LLD	VU 4yA~(MRR(MRRr,   rp  interpolation_modepadding_mode_expand_gridc           	          !"#$%&'()*+,- t        j                  dv fd       t        j                  dv fd       dt        dt        dt        ffd-dt        dt        d	t        dt        fd
+dt        dt        dt        f+fd dt        dt        dt        f -fd} j                  \  $%|j                  \  })*}|dk(  sJ r(|j                  d)*|      j                  )*d      }dt        dt        dt        f$%fd&t        j                   j                        j                  ddd      t        j                   j                        j                  ddd      dt        dt        dt        dt        f&)*fddt        dt        dt        f fd"|d   }	|d   }
dk(  r ||	%      } ||
$      }|j                         |j                         c'('dz   (}}'(dz   }}||}}||z
  ||z
  z  }||z
  ||z
  z  }||z
  ||z
  z  }|'z
  |(z
  z  }t        "fd'(|f|||f|||f|||ffD              S dk(  r< ||	%      } ||
$      }|j                         }|j                         } "||d      S  -|	%      } -|
$      }|j                         '|j                         (|'z
  ,|(z
  }s",j                  d      ,|j                  d      }dt        dt        dt        f "$%fd#dt        dt        f#'(,fd!t        !fdt        d       D              }t!        ||      S )!N)r   r#   r$   c                      d  S )NzInvalid interpolation mode r+   )rw  s   r-   r   z"_grid_sampler_2d.<locals>.<lambda>  s    -.@-AB r,   c                      d  S )NzInvalid padding mode r+   )rx  s   r-   r   z"_grid_sampler_2d.<locals>.<lambda>  s    -B<.+Q r,   coordsr$  rQ   c                 B    r|dz  dz
  n|dz  }|dz  dz
  }| |z  |z   S r  r+   )r}  r$  r   ofsr  s       r-   unnormalizez%_grid_sampler_2d.<locals>.unnormalize  s8     %2tczCs
Sj3|c!!r,   	twice_low
twice_highc                 P   ||k(  rt        j                  |       S |dz  }||z
  dz  }| |z
  j                         }t        j                  ||      }||z  j	                         j                  t         j                        }t        j                  |dz  dk(  ||z   ||z   |z
        S )Nr$   r   r#   r   )r@   r   r   fmodfloorr9   int8rj   )r}  r  r  
coords_mincoords_spancoords2extraflipss           r-   reflect_coordinatesz-_grid_sampler_2d.<locals>.reflect_coordinates  s    
"##F++]
!I-2J&++-

7K0;&--/222D{{AINEJ.j0H50P
 	
r,   c                     dk(  r| S dk(  rt        j                  | d|dz
        S r | dd|dz
  z        }n | dd|z  dz
        }t        j                  |d|dz
        S )Nr   r#   r$   rT   r   )r}  r$  coords_reflectedr  rx  r  s      r-   compute_coordinatesz-_grid_sampler_2d.<locals>.compute_coordinates  sx    1MQ;;vq$(33#6vq!tax.#Q #6vr1t8a<#P ;;/D1H==r,   c                 (     | |      } ||      S r6   r+   )r}  r$  	coords_unr  r  s      r-   compute_source_indexz._grid_sampler_2d.<locals>.compute_source_index  s    -	"9d33r,   r$   r#   r  ysc                     t        j                  d| k  t        j                  | k  t        j                  d|k  |k                    S rt   r@   r  )r  r  iHiWs     r-   in_bounds_condz(_grid_sampler_2d.<locals>.in_bounds_cond"  sF      GU&&rBw0A0A!r'2PR70ST
 	
r,   r  wsc                      | |      rndt        	
fd| j                  t        j                        |j                  t        j                        |fD              S )Nr#   c              3   p   K   | ]-  }t        j                  |d       j                         / ywr  )r@   rj   r  )r  r  r  r  r  oHoWs     r-   r  z1_grid_sampler_2d.<locals>.clip.<locals>.<genexpr>1  s7      
 KKa#((Ar26
s   36r   )r  r9   r@   r  )r  r  r  r  r  r  r  ry  r  r  r  s      @@r-   clipz_grid_sampler_2d.<locals>.clip*  sY    b"%
 A1 
ee%++e.EKK0H"M
 
 	
r,   ixiyc                 8     	| ||      \  }}}||f   |z  S r6   r+   )
r  r  r<  idx_xidx_yw_C_idxN_idxr  r  s
         r-   get_summandz%_grid_sampler_2d.<locals>.get_summand6  s0    B?ubue+,r11r,   ).r   ).r#   r   c              3   :   K   | ]  \  }}} |||        y wr6   r+   )r  r  r  r<  r  s       r-   r  z#_grid_sampler_2d.<locals>.<genexpr>L  s(      
R B"
s   c                 <     |       } |      } ||d      S r]   r+   )r  r  r:   r[   r  r  r  r  s       r-   get_value_boundedz+_grid_sampler_2d.<locals>.get_value_boundedk  s*    #B+A#B+Aq!Q''r,   r  c                     | dz
  z   } dz
  |       |       dz   |       dz   |      f}t        |      S )Nr#   r$   )rX  )r  iy_ofscsr  ix_nwiy_nwtxs      r-   	get_coeffz#_grid_sampler_2d.<locals>.get_coeffp  s[    cAg&F!%!)V4!%0!%!)V4!%!)V4	B ,B33r,   c              3   .   K   | ]  } |        y wr6   r+   )r  r  r  s     r-   r  z#_grid_sampler_2d.<locals>.<genexpr>z  s     :#y~:   r9  )r@   r   r   r  r  r  r  r  r  r   r  r  r  rV   r  rU   rX  ).r  rp  rw  rx  r  ry  r  rW   twor:   r[   r  r  ix_neiy_neix_swiy_swix_seiy_sew_nww_new_sww_se
ix_nearest
iy_nearesttyrR  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s.   ` ````                     @@@@@@@@@@@@@@@@@@@r-   _grid_sampler_2dr    s    
LLi'B 
LL	!#Q"F "# "& "

F 

s 

 

PV 


>F 
># 
>& 
>4V 43 46 4 77LAq"bZZNAr2s!8O8 yyAr2s+221aRC
6 
v 
& 

 LL188,11!Q1=ELL188,11!Q1=E

 

V 

 

4F 

 

2 2F 2& 2 2
 	VAVAQ!!R(!!R(xxz288:uqy%ueaiueu
urz*U
urz*
rEz*U
rEz* 
 t$t$t$t$	 
 
 	
 
q	 !!R(!!R(XXZ
XXZ
:z155BB

%Z%ZaBaB	(& 	(f 	( 	( 	(
	43 	46 	4 	4 :q::'33r,   c                 "    t        | ||||      S )N)rp  rw  rx  r  )r  )r  rp  rw  rx  r  s        r-   grid_sampler_2dr  ~  s      	-!# r,   c                 &    t        j                   j                         dk(  xr j                         dk(   fd       t        j                   j                  d      j                  d      k(   fd        z  j	                  d      S )Nr$   r#   c                  L    d j                          dj                          S )Nzmatrix @ vector expected, got r  r   r|   r  s   r-   r   zmv.<locals>.<lambda>  s!    0BswwykJ r,   r   c                  v    d j                  d       d j                  d       dj                  d       dS )Nzsize mismatch, got input (r   r:   r#   z), vec (r4  r  r  s   r-   r   zmv.<locals>.<lambda>  s<    ,TYYq\N!DIIaL>RURZRZ[\R]Q^^_` r,   r   )r@   r   rP   r$  r   r  s   ``r-   r  r    ss     
LL
a*CGGINJ 
LL		!#` 3J""r,   c                     |-|dz
  |z  dz   }d|z
  | z  |t        j                  |       z  z
  }nd|z
  | z  t        j                  |       z
  }|||z  }t        ||      S r]   )r   
logsigmoidr   )r|   r   r   
pos_weightr   
log_weightr   s          r-    binary_cross_entropy_with_logitsr    sv    
  1n.2
F
d"j1<<3E&EFF
d"Q\\$%77f}i00r,   tensor1tensor2is_outc                   	 | j                   |j                   k\  r| |fn|| f\  }}ddlm	 |j                   dk\  r|j                   dk  sy|j                  r|sy| j                   dk(  ry 	|j	                         dk(        ry|j
                  }|j                         }dg}t        |dd        D ]  }|j                  ||d   z          t        	fd	t        |t        t        |            |      D              S )
Nr   rJ  r   r$   FTr#   rT   c              3   V   K   | ]   \  }}} |d k(        xs  ||k(         " ywr  r+   )r  r   r   r$  rK  s       r-   r  zshould_fold.<locals>.<genexpr>  s9      D% 	tqy!B^DEM%BBs   &))ru  rk  rK  r  r   r  rn  r  rm  r   r  rm  )
r  r  r  t1t2t1_shape	t1_strideexpected_strider$  rK  s
            @r-   should_foldr    s     $+<<7<<#?gwgwEWFBDGGqLRWW\	||qbhhjAo&xxH		I cO!"& ;tob&99:; !$tH_56"
  r,   )pass_is_out)r  c                n	   ddl m}m} | j                         }|j                         }|dk7  r|dk7  sJ |dk(  r|dk(  rt	        j
                  | |      S |dk(  r|dk(  rt	        j                  | |      S |dk(  rC|dk(  r>t	        j                  t	        j                  t	        j                  | d      |      d      S |dk(  r|dk(  rt	        j                  | |      S t        | ||      r>||kD  }|r|j                  n| }|s|n|dk(  r| j                         n| }	|j                  }
t        |
d d       }t        t         j"                  |      }|	j                         dk(  }|r|j%                  |	j                  d          |j'                  ||
d         }|rWt        j(                  j*                  j-                  |j                  |	      |      }|r|j                  j/                         S |S t        j(                  j*                  j-                  |j                  |	      |      S |dk\  rp|dk\  rj|dkD  r| j1                  d      nd}| j1                  d      }| j                  d d }|dkD  r|j1                  d      n|j1                  d      }|dkD  r|j1                  d      nd}g }t3        |dz
        D ]"  }|j%                  |j1                  |             $ |dk(  r|dk(  r{ ||d   |d   k7        rj ||d   dk(        r'| j4                  rt7        | j                  d      |      S  ||d   dk(        r'|j4                  rt7        | |j                  d            S t        t	        j8                  ||            }|||gz   }t;        |      }| j=                  |      j'                  |||      }|dk(  }|r7||gz   }|j=                  |      j'                  ||      j                  d      }n)|||gz   }|j=                  |      j'                  |||      }|}|dkD  r|j%                  |       |dkD  r|j%                  |       |r/|j?                  |      j                  d      jA                  |      S |j?                  |      jA                  |      S t	        jB                  dd	        y )
Nr   )rK  guard_or_truer#   r$   rT   rQ  r   Fc                       y)Nz/both arguments to matmul need to be at least 1Dr+   r+   r,   r-   r   zmatmul.<locals>.<lambda>J  r  r,   )"rk  rK  r  rP   r@   dotr  r  r  rV   r  rV  r  r  rm  r   r  r   rm  r  r  r   _unsafe_viewr  r$  rU   r  rU  broadcast_shapesr  r  bmmr  r   )r  r  r  rK  r  dim_tensor1dim_tensor2r  r  r  sizes_1output_shapefolded_dim1t2_is_matrix	t1_foldedr  r  m1batch_tensor1m2rI  batch_tensor2rg  expand_batch_portiontensor1_expand_sizeexpand_batch_producttensor1_expanded
vector_rhstensor2_expand_sizetensor2_expandeds                                 r-   rU  rU    sn    T++-K++-K!q 000aK1,yy'**		kQ.xx))		kQ.}}UXXeoogq&A7KQOO		kQ.xx))	Wgv	.  +-	$WZZ'$G+:J799;PW 	 ((GCRL)X\\<8 vvx1}, JJ{GBK8	 YY^^00b1A<PF-6699'')BFB99>>..y||B/?NN		kQ. !,aGLLQ\\"cr*!,qW\\"gll26F +aGLLQ#%{Q' 	2A  a1	2 1q mA.-2BBCmA.!349N9Ngooa0'::mA.!349N9Ngwq'9::  $""=-@ 
 3aW<#$89 #>>*=>FF !R
 !A%
"6""=23-r21  #7"a"@&~~.ABJJ$b!  ,?"?"#''(89AA"EJJ<XX#''(89>>|LLUUVr,   r  r  c                 \     j                   \  }}t        |d   ||      }t        |d   ||      }t        j                   t        j                  j
                        \  }}t        j                  |d    j                        j                  |      }	t        j                  |d    j                        j                  |      }
t        ||
|      }t        ||	|      }|j                  d      }|j                         }|j                         }||z
  j                  dd      }||z
  j                  dd      }|j                  t        j                        }|j                  t        j                        }|dz
  ||dz   |d	z   f}|dz
  ||dz   |d	z   ft        |      t        |      }d
\  } j                   t        j"                  k(  rt%              t%        |      }D cg c]@  }|dz  z  t        j&                  |      dz  z   j                  t        j(                        B c}|D cg c]@  }|d|z  z  t        j&                  |      dz  z   j                  t        j(                        B }} fd fdt+        fd|D              } j                   t        j"                  k(  r|J t-        |||      }nt/        d t1        ||      D              }t        j2                         }|j5                  |      }|S c c}w c c}w )Nr   r#   rL  r  r   rT   r   rh   r$   rZ  r   c                     t        j                  | ddz
        }t        j                  |ddz
        }t        j                  d d ||g      }|S r  )r@   r   r   rP  )r  r  y_idxx_idxr"  in_hin_wr   s        r-   load_boundedz0upsample_bicubic2d_default.<locals>.load_bounded  sO    B4!8,B4!8,utT5%&@Ar,   c                      t         fdD              }j                  t        j                  k(  rJ t	        |      S t        d t        |      D              S )Nc              3   0   K   | ]  } |        y wr6   r+   )r  x_ofsr  r[   s     r-   r  zCupsample_bicubic2d_default.<locals>.get_x_interp.<locals>.<genexpr>  s     Bl1e,Bs   c              3   ,   K   | ]  \  }}||z    y wr6   r+   rV  s      r-   r  zCupsample_bicubic2d_default.<locals>.get_x_interp.<locals>.<genexpr>  s     JRBGJr  )r  r   r@   r<  r  r  r  )r[   src_xr   ixs_ofsr  weights_precision_x	weights_xs   ` r-   get_x_interpz0upsample_bicubic2d_default.<locals>.get_x_interp  sW    B'BB;;%++%&222%eY8KLLJCy4IJJJr,   c              3   .   K   | ]  } |        y wr6   r+   )r  y_ofsr  s     r-   r  z-upsample_bicubic2d_default.<locals>.<genexpr>  s     ;%,u%;r  c              3   ,   K   | ]  \  }}||z    y wr6   r+   rV  s      r-   r  z-upsample_bicubic2d_default.<locals>.<genexpr>  s     L(2rb2gLr  rO  )r  r  rE   rF   rN  r  r@   r  r  r9   r   rV   r  r   r  rQ  r   r<  r  r   int16r  r  r  r  r   r  )r   r  r  r  r  rW   h_scale_factorw_scale_factorr   rg  r  x_floaty_floatr:   r[   yscaler#  iys_ofs	weights_yweights_precision_yr<  src_yr  rP  r  r  r  r  r  r   r  s   `                       @@@@@@@r-   upsample_bicubic2d_defaultr  M  s    {{Aq$ $D+a.-QN#D+a.-QN''5#H#H#U#UHAu 	[^ELL9<<5<IA[^ELL9<<5<IA#NA}EG#NA}EG#GAA k  c*Fk  c*F	U[[A	U[[A1uaQA&G1uaQA&G08I08I/9,,{{ekk!7	B7	B 
 !**+ejjmc.AAEEekkR
	 
 !**+ejjmc.AAEEekkR
	 

K K ;7;;E{{ekk!"...#E96IJLc%6KLL //6M];FMA

s   9AL$AL)c                 $   t        j                  t        |      t        |      z   dk(  d        |H|J t        t        t
        t
        f   t	        d t        | j                  dd  |      D                    }|r|nd\  }}t        | ||||      S )Nr#   c                       y)Nz:Must specify exactly one of output_size and scale_factors.r+   r+   r,   r-   r   z(upsample_bicubic2d_vec.<locals>.<lambda>  r  r,   c              3   P   K   | ]  \  }}t        t        |      |z           y wr6   )r   r   )r  r<  ro   s      r-   r  z)upsample_bicubic2d_vec.<locals>.<genexpr>  s*      Au 	!u,-s   $&r$   rZ  )	r@   r   r  r   r  r  r  r  r  )r  r  r  r#  r  r  s         r-   upsample_bicubic2d_vecr    s     
LL[D//14L (((#s(O  #AGGABK ? 
 )6}<GW%amWgVVr,   c                 (      fd}t         ||      S )Nc                     t        j                  |  ||z   j                        }|dz
  |dz
  |j                         z
  j                         z
  S )Nr  r#   )r@   r  r  r   r   middler   dim_idxr  s       r-   r  z_reflection_pad.<locals>.idx  sF    ,,ufunQXXFzVaZ'++-7<<>>>r,   _reflection_or_replication_padr  r  r  s   `  r-   _reflection_padr    s     ? *	 r,   c                 (      fd}t         ||      S )Nc                     t        j                  |  ||z   j                        }t        j                  |d|dz
        S )Nr  r   r#   )r@   r  r  r   r  s       r-   r  z_replication_pad.<locals>.idx  s6    ,,ufunQXXF{{7Avz22r,   r  r  s   `  r-   _replication_padr     s     3 *	 r,   idx_fnc                 n   t        |      dz  t        j                  | j                         dz   dz   fv fd       | j                   d  }| j                         z
  }t              D cg c]  }|ddz
  |z
  z      }}t              D cg c]  }|ddz
  |z
  z  dz       }}| }t              D ]E  }d g|j                         z  }	 |||   ||   ||         |	||z   <   t        j                  ||	      }G t        j                  |      }
|j                  |
      }|S c c}w c c}w )Nr$   r#   c                  (    d  d dz    d dz    dS )Nreflection_padzd requires r#   zD or r$   zD inputr+   r   s   r-   r   z0_reflection_or_replication_pad.<locals>.<lambda>  s$    .[q	sQwiwO r,   rO  )rz  r@   r   rP   r  rU   r   rP  rE   r   r  )r  r  r!  	inp_shapenc_dimrg  padding_leftpadding_rightr  r  rP  rP   s              @r-   r  r    sH   
 g,!
C	LL	C!GS1W%%O IUUWs]F8=c
C1GAq1-.CLC=B3ZHWQ#'A+.23HMHF3Z 1&**,. a)A,a@PQAJ##FC01 //7M];FM DHs   1D-D2c           
          t        |      dz  |j                   d  D cg c]  }|dz
  	 }}t              D cg c]  }|ddz
  |z
  z      }}t              D cg c]  }|ddz
  |z
  z  dz       }}g }t        |j                        D ]c  }dg|j                  z  }	d|	|<   |j	                  t        j                  |j                  |   |j                        j                  |	             e |d   | d  }
d t              D cg c]  }|
|   ||   z    }}t              D cg c]  }||   |
|   z
   }}t              D cg c]  }d||   z  ||   z   |
|   z
   }}t              D cg c]  }||   d||   ||   z   ||   z   f }}t        j                  t        j                  t              D cg c]  } ||          c}      }t        j                   ||z   d      } fd}t        j                  t              D cg c]  }g d	 c} D ]  }|t!        dgz        k(  rg }g }t              D ]t  }||   dk(  r||   }||   }n=||   dk(  r||   }|
|   d||   f}n$||   dk(  r||   }|
|   ||   ||   z
  ||   dz
  f}|j	                         |j	                         v  ||||      } |S c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w )
Nr$   r#   rT   r  c                 F    | \  }}}t        j                  ||k\  ||k        S r6   r  )index_rangerg  lbubs       r-   index_range_conditionz7_reflection_pad_backward.<locals>.index_range_condition  s(    	2r  b!r'22r,   r   r   c           	      .   t        	      D ]*  }||   d   ||   d   k  }t        |t              s%|s(| c S  t        j                  t
        j                  |D cg c]
  } |       c}      }t
        j                  
||z   d      }| |z   S c c}w )Nr$   r#   r   )rU   r8   r  rM   r   r   r  r  )r   r   index_rangesrg  upper_less_than_lowerr+  r  gr  rP   rm   r.  s           r-   r  z,_reflection_pad_backward.<locals>.accumulate/  s     s 	A$0OA$6a9K$K!/6;P	
 COPK";/P
 %%k4S#Fax Qs   B
)rT   r   r#   )rz  r  rU   ru  rm  r@   r  r  r  rM   r   r   r  r  	itertoolsr
   r  )rm   r:   r  r`  dhwrg  r'  r(  r  
view_shapexyzcenterleft_reflectright_reflectrange_cr  r   r  rW   areaoutsr0  r   r+  r  rP   r.  s   `                       @@@r-   _reflection_pad_backwardr=    s`   
 g,!
C''3$%.
)Q1q5
)C
)8=c
C1GAq1-.CLC=B3ZHWQ#'A+.23HMHG166] SS166\

1u||AGGAJqxx@EEjQRS
 	#A
3$%.C3 16c
;1c!f|A&;F;6;CjALOc!f,ALADI#JOqQQZ,q/1CF:OMO
 NSSVZHIAs1vQ/-2BBCG  eCjQ0<QD $$[$F
CHD !!c
#C1J#CD 45!s##s 	-AAw!|Qi%ajaB"1o"1vq,q/:aA#A&"1vs1va0@'@#a&1*MKK,	- $l3-40 Kc *CH2 <AO R& $Ds5   K K$$K)K.8K3K8K=L
&Lr   r   r   c                j    t        j                  | ||      }t        j                  | ||      }||fS )Nr   )r@   aminrP  )r|   rP   r   r?  rP  s        r-   aminmaxr@  Y  s2     ::dW5D::dW5D:r,   r   c                    t         j                  t        j                  t        j                  |       d|       |||      S )Nr   r   )r   r   r@   rj   isnan)r|   rP   r   r   s       r-   nansumrC  c  s2     88EKKD 11d;S'QV8WWr,   r   r5  r  rO  r5  c          	      N    t         j                  j                  d| d||||      S )Nr   r#   rD  r   r  
start_step)rb  r   r5  r  rO  s        r-   arange_defaultrH  i  s/     ;;!!	3vf "  r,   c          	      N    t         j                  j                  | |d||||      S )Nr#   rD  rF  )ra  rb  r   r5  r  rO  s         r-   arange_startrJ  x  s/     ;;!!sAU6&Z "  r,   c                      ddl m}  || i |S )Nr   )out_dtype_dense)!torch._higher_order_ops.out_dtyperL  )rG   rH   rL  s      r-   out_dtype_decomprN    s    AD+F++r,   marginc                 t   	
 t        j                          t        j                         j                  d   
 j                  d   	t        j                  |dk(  xs |dk(  d        t        j                   j
                  dk(  xr 	dk7   fd       t        j                  j
                  dk(  xr j                         
k(  
fd       Qt        j                        t        j                  j
                  dk(  xr j                         	k(  	fd       j                  d      t        j                   d      }||z
   z   }|j                  d      }|dk(  r|n||z  }|   z  }t        j                  	 j                  	      }t        j                  |k7  |d      }|t        j                  j                  k(  r|j!                         S |t        j"                  j                  k(  r |j%                         |j                  d   z  S |j!                  d
      S )Nr   r#   r$   c                       y)Nz only p == 1 and p == 2 supportedr+   r+   r,   r-   r   z#multi_margin_loss.<locals>.<lambda>  r  r,   c                  "    d j                    S NzMExpected non-empty vector or matrix with optional 0-dim batch size, but got: r  r  s   r-   r   z#multi_margin_loss.<locals>.<lambda>  s    _`e`k`k_lm r,   c                  (    d  dj                    S )Nz#inconsistent target size, expected r  r  )nframer   s   r-   r   z#multi_margin_loss.<locals>.<lambda>  s    5fXYv||nU r,   c                  (    d  dj                    S )Nz#inconsistent weight size, expected r  r  )rP   r   s   r-   r   z#multi_margin_loss.<locals>.<lambda>  s    9#i~V r,   r  r  r   )r@   
atleast_2d
atleast_1dr  r   ru  r   rV   r;  rW  r  r  rj   r"   r)   r}   r   r*   r   )r   r   rI  rO  r   r   urk   r  rP   rU  s   ``  `    @@r-   multi_margin_lossrZ    s    U#Ef%F[[^F
++a.C	LLa!16#MN	LL

a$C1Hm 
LLq5V\\^v5U !!&)KK163!6V	
 a FU0A
UA	AA!VQAv
,,s5<<
0CC6M1a(AINN(((vvx	imm))	)uuw##vv!v}r,   	is_targetc                    | j                   |j                   t        j                  |       } t        j                  |      }| j                   d   }t        j                  t	              dk  xr |dk7  fd       t        j                  t	              dk  xr k(  fd       t        j
                  ||j                        }|dk(  }t        j                  t        j                  |||      dd	      }||k  }t        j                  ||d      }t        j                  | d|
      }	t        j                  ||d      }
t        j                  ||
j                  d      k(  d      }d|	j                  j                  d      z
  | z   }|j                  d      }||z  }t        j                  |d|      }|t        j                  j                   k(  r!|j#                  d      j%                         }n@|t        j&                  j                   k(  r|j#                         }n|j#                  d      }|j)                  | j*                        j-                        }||fS )Nr#   r$   r   c                      d  S rS  r+   )orig_input_shapes   r-   r   z0multilabel_margin_loss_forward.<locals>.<lambda>  s    _`p_qr r,   c                      d d  S )Nzinconsistent target size: z for input of size: r+   )r^  orig_target_shapes   r-   r   z0multilabel_margin_loss_forward.<locals>.<lambda>  s    ,->,??STdSef r,   r  rT   Tr   r  r   rh   )r   rT   )r  r@   rW  r   rz  r  r  r?  rj   r;  anyrV   TrW  r"   r)   r}   r   r   r*   r9   r   r  )r   r   r   rP   r  is_endend_idxtarget_masktidx0rY  tidx1r[  rk   r^  r`  s                @@r-   multilabel_margin_loss_forwardrh    s    {{U#Ef%F
++a.C	LL"/saxr 
LL!#M(9=M(Mf
 ,,s6==
1Cr\FjjVS#6BMG-KKKVQ/EU%0AKKVR0E		#R!88a@Iaccmmm##e+A	AA	CAIq!$AINN(((EEgE##%	imm))	)EEGEEgEU[[)112CDIi<r,   )	attn_maskro   querykey	dropout_p	is_causalri  c                    t        j                  t        j                          fd       t        j                   j                         dk(  xr( j                         dk(  xr j                         dk(   fd       t        j                  dk(  fd       t        j                   j                  d   j                  d   k(  xr j                  d   j                  d   k(  d        t
        j                  j                   ||d | j                  d      j                  d      k7  		      \  }}|j                  d
ddd      j                  t         j                        j                  dd
dd      }||fS )Nc                  "    d j                    S )Nz-query must be FP32, FP64, BF16, FP16 but got r   )rj  s   r-   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>	  s    ?}M r,   r9  c                  n    dj                          d j                          dj                          S )Nz,q, k, v must be a 4 dimensional tensor, got r  r   )rk  rj  r}   s   r-   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  s3    >uyy{m2cggi[XZ[`[d[d[fZgh r,   r   c                      d  S )Nz&dropout probability must be zero, got r+   )rl  s   r-   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  s    $J9+"V r,   r   c                       y)Nz&q, k, v should have the same head sizer+   r+   r,   r-   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  r  r,   r#   )ri  rl  rm  dropout_maskro   
enable_gqar$   r   rO  )r@   r   rC  rP   r  r   "_scaled_dot_product_attention_mathr  r$  r  r  rT  )	rj  rk  r}   rl  rm  ri  ro   r  attns	   ````     r-   *scaled_dot_product_flash_attention_for_cpurw    sV    
LL&M 
LL		q@SWWY!^@		q0@h 
LLSV 
LLA%++a.(KSYYq\U[[^-K8
 ::BB::a=CHHQK/ C 
LFDF 	q!Q"	%"9"9	:	Aq!	 
 4<r,   c                 .    t        |       fd       }|S )Nc                  <     | i |}| d   j                  |      S rt   )r  )rG   rH   r   outplace_ops      r-   
inplace_opz$register_inplace.<locals>.inplace_opB  s%    4*6*Aw}}S!!r,   r   )aten_oprz  r{  s    ` r-   register_inplacer}  A  s"    G$" %" r,   c                 B   | j                         s&| j                         st        |      }t        |      }t        j                  ||      }t        |t        j                        r|dk7  r||z  }|dk(  r|S t        |t        j                        r|dk7  r| |z  } | |z   S r*  )rC  r  r  r@   r  r8   numbersNumber)r|   batch1batch2rd   rn   r  s         r-   baddbmmr  J  s     !!#DOO,=4yE
YYvv&FeW^^,
%qydGNN+tqyd{&=r,   c                 2    t        j                  | |d      S )Nr  r  r  )r|   rH  s     r-   floor_divider  \  s     99T588r,   c                 `    t        j                  t        j                  | j                  d      S r]   )rM   r   r  r   r  )r  s    r-   	sym_numelr  b  s    HLL!''155r,   r   r   c                    |"t         j                  j                  | g |      S t         j                  j                  | g ||      S )Nr   r  )r   r   dim_IntListIntList_out)r|   r   r   s      r-   sum_defaultr  g  sC     {xx##D"E#::xx##D"Es#CCr,   c           	          t        | t        j                        s| S |@t        j                  j                  | t        t        | j                                           S t        j                  j                  | |g      S r6   )	r8   r@   r   r   r  dimsrm  rU   rP   )r|   rP   s     r-   squeeze_defaultr  t  s\     dELL)
{||  tE$((*,='>??||  u--r,   c                 2   t        fdt        t        | j                              D              }|j                  t
        j                  k(  rt
        j                  nd }| j                  d|d|      }| ||j                  |j                        z  z  |fS )Nc              3   .   K   | ]  }|k7  s	|  y wr6   r+   )r  rg  rP   s     r-   r  z)_weight_norm_interface.<locals>.<genexpr>  s     @1qCxQ@s   
r$   T)r   r   )
r  rU   rz  r  r   r@   r  r   r   r9   )r"  r2  rP   keep_dim
norm_dtyper   s     `   r-   _weight_norm_interfacer    st     @c!''l 3@@H !5>> 9tJ66!Xt:6>DDGGAGG$$%t++r,   assume_uniqueinvertc                   t        | t        j                        s!t        j                  | |j                        } t        |t        j                        s.|rt        j
                  | |      S t        j                  | |      S |j                         dt        | j                         d      z  k  rt        | ||      S t        | |||      S )Nr  g      $@g(\?r  r  )r8   r@   r   ro  r  ner   r   rR  isin_defaultisin_sorting)elementstest_elementsr  r  s       r-   isinr    s     h-<<1E1EFmU\\288Hm4488Hm44tc(..*:E&BBBHmFCCm=
 	
r,   )rM  c                B   |?t        j                  | j                         t         j                  | j                        }n?t        j                  | j                         |t         j                  | j                        }|| k  j                  | j                        }|S )Nr  )rM  r   r  )r@   randr$  r   r  r9   r   )r|   rM  raw_prI  s       r-   	bernoullir    sq     

499;emmDKKP

IIK--;;	
 
$**%AHr,   r  c                L   | j                         dk(  r%t        j                  | t        j                        S | j                  d|j
                  z  z   }| j                  |      }t        t        d|j
                   dz
  d            }||k(  j                  |      }|r| S |S )Nr   r   )r#   rT   r#   r   )
r   r@   
empty_liker  r  ru  r  r  rU   ra  )r  r  r  expanded_elem_shaper:   rP   rF  s          r-   r  r    s    ~~1

;;"..4-2D2D+DD)*A
b=---126
7C
"
"s
"
+CC4"s"r,   c                   | j                         }|j                         }|rt        j                  ||g      }t        j                  |d      \  }}|dd  |d d k(  }	t        j                  |	ddgd      }	|r|	j                         }	t        j                  |	      }
|
j                  d||	      }
|
d| j                          S t        j                  |      \  }}t        j                  ||      }t        j                  ||j                         k  |d      }||   |k(  }|r|j                         n|}|j                  | j                        S )NT)stabler#   rT   r   F)r3  r@   r&  sortrk  logical_notr  r  r   searchsortedrj   r  r  )r  r  r  r  elements_flattest_elements_flatall_elementssorted_elementssorted_orderduplicate_maskr  sorted_test_elementsrW   r  test_idxcmps                   r-   r  r    s?   $$&M&..0 yy-1C!DE(-

<(M%(,0DD..~1vuM+779N/q,?A())"'**-?"@a  !5}E;;s%9%?%?%AA3J"8,=#)coos{{8>>**r,   c                 .    | j                  d      }||   S rS   )r  )r|   r  	flatteneds      r-   taker    s     R IUr,   c                     |t         j                  }|t         j                  k(  rt        |      }t        j                  | |j                  |      S r*  )r@   rT  preserve_formatr   r   resizer  )r|   rH  rP  s      r-   	resize_asr    sD    //----e4;;tU[[;FFr,   )FF)noner6   )r$   )r   NNr#   )rT   FFr  r  rd  )r#   r#   F)Fr   )r   rh   N)r   r#   N)FrZ  )NNN)r   r   FT)r   r   Fr!  )r   F(  rM   r3  r  r  ro  collections.abcr   r   
contextlibr   enumr   r   r   r	   r
   typingr   r   r   r   r@   torch._meta_registrationstorch._primsrV  r  torch._prims_common_prims_commonrE   torch.nn.functionalr  r  r   r   r   r   torch._decompr   rM  r   r   r   r   r   r   torch._prims_common.wrappersr   r   r   r   torch.utilsr   rC   torch.utils._pytreer   r6  DispatchKeyr    rm  str__annotations___opsr  r   r"   rN  r  rO   rO  compute_only_pw_cast_for_opmathpw_cast_for_opmath"pw_cast_for_opmath_non_tensor_argsr  pw_cast_for_int_to_realr  rX   ra   rc   rl   r   rx   r3  r  r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   r)   r}   r   _safe_softmaxr  r  r
  r  rY   r  r  r   r  r!  r.  r7  r:  r@  rC  rE  rG  rK  r_  rg  slicer{  r  rf  r  r  r  r  r  r  r  r(  r,  r5  r;  r?  py_implCompositeImplicitAutogradAutogradr>  rS  rW  r]  re  r  rs  rv  rz  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r  r$  r(  unsafe_chunkr+  r'  r&  no_statsr/  r3  r?  rE  rI  rK  _fused_dropoutrN  r[  r  rP  r  lift
lift_freshr^  rc  rf  ri  rh  r|  r  r  _adaptive_avg_pool2dr  r  r  r  r  r  r  r  r
  r  r  r  r  	Generatorr  r%  r(  r>  r  rE  rK  r.  _upsample_nearest_exact1drG  rM  r3  r9  r@  r+  rX  ra  re  ri  rw  r  r  r  r  r  rnn_tanhr   r  rnn_relur  r  r  r  r  r  r  r  lstmr  r  r  r  grur  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r'  r)  r  rP  r  r  r9  r?  rB  rD  rH  rK  rQ  rX  r  r^  ri  rl  rq  rs  rv  r  r  r  r  r  rU  upsample_bicubic2dr  r  reflection_pad1dreflection_pad2dreflection_pad3dr  replication_pad1dreplication_pad2dreplication_pad3dr   r  reflection_pad1d_backwardreflection_pad2d_backwardreflection_pad3d_backwardr=  r@  rC  r  rS  r5  rH  ra  rJ  rN  rZ  rh  +_scaled_dot_product_flash_attention_for_cpurw  r}  r  r  r  r   r  r  rP   r  r  r  r  r  r  r  r  addbmm_addbmmaddmm_addmv_baddbmm_fill_gelu_r  
hardswish_	hardtanh_hardtanhhardsigmoid___iand____and____ilshift__
__lshift__r  r  index_reduce_index_reduce__ior____or____irshift__
__rshift____ixor____xor__leaky_relu_
leaky_relulogit_logitrelu_r  renorm_renormround_r  scatter_r  scatter_add_scatter_addscatter_reduce_scatter_reducesilu_r+   r,   r-   <module>r     s<N       
 . "  % $ - -     #   , , 0 7   * ( hh"" c zz~~   %$)	''99' ' "	'T #*88@@# 
 uDDLL  &-88@@ & "
 "uDDQQ  c f  **+\2F 2v 2   ,2 --.\4v 4& 4   /4 ../\S S6 S S5 S   0S
 ))*\


 
 	

 
 
   +
2 ))*+( ,( ))*+"V " ," (()@f @ @   *@ 112\f F    3 ../\PP%P05P@EP  0P 'GF Gv G   (G //0F & V    1 //0\:F :& :U :  1: 001\LL%L7<LNRL   2L **+\) )f )3 )   ,)< **+5v 5f 5  ,5 		"&v && &   #& **+\>v >V > >   ,>
 **+6 6 66 6 ,6 334%%
% % 66>	% 5% 667



 
 	

 
 
 
 
   8
$ 112\<f <F <F <v <   3<v #   &3<>>3G3G1
1 1-011   '1 ../\11 &1061CF1   01 **+0 ,0 ++, ^^))		1
	1	1 	1 		1   -	1 44<<=%/5BEMR  > 44??@
R
R

R 
R 	
R
 
R 
R  A
R 00889		%	/5	BE	NS	  :	 00445
R
R

R 
R 	
R
 
R 
R  6
R$$
$ $ V	$
 $ $ $ $@ ))*\Nf NF N N N   +N$ ../\''
' ' V	'
 ' ' ' '  0'T 001\
  V	
      2B 112  $^^))	1
11 V1 	1
 1   31& ::;\
  $^^))
  V	
     <  --. ^^))111 1 	1   /1 667\
 ^^))			
	 	 		
 	   8	 		") )v )% )  #) ,,-& &F &v &  .& ++,	O	Oc	O 
	O 		O
 
	O 	O  -	O 

))* 3?
3? 
3? C=	3?
 
#3? 3? +3?l (4<SM
38_. **+ $U$U	$U 
$U C=	$U
 
#$U $U  ,$UN ,,-E Ed3i Ec ERU E  .E
 ../OO&*3iO9<ODGOORO  0O%+:?++ 334\ XX!'X.1X@EX !  5X 778 KK!'K.1K@EK !  9K*& $MMcM 3iM #Y	M
 IM M  %M` $]]c] c] 3i	]
 #Y] I] ]   %]@ 445
 
v 
e 
  6
 ,,-Y
Y"3iY47Y?BYJMYY  .Y" ++334>B

%
,4UO

  5
& %kCCDk2236 e HTN  4 E & ++,VVA& AU A8D> A  -A & S    '. ))*F  T   +0 ' $  	
    (& 556  	
   7:DI &\	  
&\	*T&\ 
tF| 
# 
"&\"	" "J 00$//2E2EFG
 !	&\	  
&		
  H  	'')C)C)G)GH "&	
c 
 
$v,		
 d6l  ))0015 5C 5c 5%PSBT 5 25 44<<=67BB $S	B03B
63;B >B 

))*/ /C /c /%:L / +/" --55)) &/
&/ &&/ 
&/ 63;	&/&/T 

# f F # #    $" ../
 

  	
     0" 

#	 	f 	6 	 	 	   $	 77??@S&S&S& S& 	S&
 FS& S& S& 
S& S& dS& 8FXf-x/??@S&  AS&n 77;;<  	
 F   
  d ,, ,, ,, 8FXf-x/??@ =88F# x/?  77??@OOO 3iO 	O
 O VO 6
O dO 8FXf-x/??@O AOf 77;;< 3i 	
  V 6
 d ,, ,, ,, 8FXf-x/??@ =4 ,,445333i3 V3 
%	3
 66>3 63l 55==>GGG 3iG 	G
 VG dG 8FXf--.G ?GTOOVO 6
O 6"	O
 &!O O O 
O O 6668F#3Xf5EEFOd ../UK/((V( 6
( 6"	(
 &!( ( ( 
( 666!"( 0 0(4 ''(<(<=''(M(MN 
 
V 
 6
 
 6"	 

 &! 
  
  
 
 
 666!" 
 O > 
F "";#H#HIG4< G JG AAIIJV 6
 	
   
 666!" K* 55==>((V( 6
( 	(
 ( ( ( 
( 666!"( ?(  55>>?((V( 6
( 	(
 ( 
( 666!"( @( @@HHIKKVK 6
K 	K
 K K K 
K 6666612K JK0V 6
 	
  
  : 44<<=11V1 6
1 	1
 1 1 
1 6666)*1 >14 ??GGHCCVC 6
C 	C
 C C 
C 6666669:C IC4 22::;11V1 6
1 	1
 1 1 
1 6666)*1 <14 ++,VV   - & $(%)37&VZ & EKK &
 U\\"& & & E//0&  '&X diiAB  C &&{';';<--.VVVV, 6
 6"	
 &!  !&  - / => 00889 V 6"	
 &!  &!  
 d  68F#Xf%556 :6 77??@ddd Vd 6"	d
 &!d d &!d d 
d dd 68F#Xf%556d AdP 77;;<""" V" 6"	"
 &!" " &!" " 
" d" ,," ,," ,," 68F#Xf%556" ="J 778VVV$  6"	
 &!  v  % 90 667VVV$  6"	
 &!  v   % 82 112c'v c'E#s(O c'   3c'L
)8<S	HK0 ))**8
*8*8 c*8  +*8Z ))*494949 c49 I	49
 #Y49  +49n ( HH	H H 	H H )H ' II	I I 	I I  (I$ %@%@	%@ %@ 	%@ %@ %@P ))112"";#H#HI J 32 (()<: <C <
 <J < *< (=* =3 =z =: =  )=@@@$.@8B@PT@, 001Xx (f (vv~)> (  ! 2( % $'$'+/		tS%	  c5 
! (	  &  &; ';
8 //334//334//334$$[%J%JK$$[%9%9:$$[%J%JK$$[%9%9:$$[%J%JK$$[%9%9:	3	3$s)$	3 DK(	3 		3 ; L ; L ; L 5 5 5	3 66::;66::;66::;##++K,Q,QR##++K,@,@A##++K,Q,QR##++K,@,@A##++K,Q,QR##++K,@,@A	?	?$s)$	? DK(	? 		? B S B S B S < < <	?@ 0088$:Q:Q:U:UVW  (()N)NO  (()=)=>Dd; #;;c; UO; 	; < ? P X; 	##++T-K-K-O-OP ''//0U0UV''//0D0DEDd; #GGcG UOG 	G < F WG 0088$:Q:Q:U:UVW  (()N)NO  (()=)=>Dd; !% $	GGcG uoG uo	G
 G < ? P XG 	##++T-K-K-O-OP ''//0U0UV''//0D0DEDd; !% $	SScS uoS uo	S
 S < F WS 0088$:Q:Q:U:UVW  (()N)NO  (()=)=>Dd; !% $ $QQcQ uoQ uo	Q
 uoQ Q < ? P XQ 	##++T-K-K-O-OP ''//0U0UV''//0D0DEDd; !% $ $		c	 uo	 uo		
 uo	 	 < F W	 
 	c %! 	
  4 >/ FK+\&,/-d& R ++,[BBC[112. 3 D -.8 ++,[BBC[112. 3 D -.8 **+KAABK001. 2 C ,.@ **+KAABK001. 2 C ,.@/6=@/d 		(>>?--.S / @ )S> 		'==>,,-S . ? (S<;; &{<<={++,. - > '.6 '==>,,-. . ? (.6 44889!!))+*O*OP!!))+*>*>? @ Q : 33778  (()N)NO  (()=)=> ? P 9 0044511556##K$I$IJ##K$8$89%%k&K&KL%%k&:&:;&&{'L'LM&&{';';<A = N < M : K 7 6A //779O9O9S9STU
 !%	KKcK K uo	K
 K  VK 	%%t'?'?'C'CD !!))+*>*>?
 !% $UUcU U uo	U
 uoU U  @
U 	&&(A(A(E(EF 
 !% $ $

c
 
 uo	

 uo
 uo
 
 
V/7	&	7$,V$47IO77-'9 -f - EEcE E %!	E
 E ER ))112F v $  3 ,,d.?.?@A  B ++,-" ." //019 29 2234C 5C6 AABCM DM23 
3 3  V3  	3 
 3  66>3 l --.X~&L
LL VL 	L
 L 66>L ' /L6 //0X~&L
LL VL 	L
 L 66>L ' 1L/F /u / /5F 5u 5 5
 
3E 
(F%7 FV F F!Xf% !& !NN#'N05NEJ\\N&f & & &T &"/f / / / /T /"!V !49 !T !$V $49 $T $ 223S StCy S S   4S  c4c4
c4 c4 	c4
 c4 c4 c4L ,,-  
  	
     .   	#   !	# ==>$)..:N:N1  ?1  d t @ [BBC>>?', vW  @ DvWr 0088$:Q:Q:U:UVW  (()=)=>
  $#PPsCxP P e_	P
 e_P P   ? XPf //334$$[%J%JK$$[%9%9:
 48	WW%S/*W W E%,/0	W
 W   ; L 5
W. --.--.--.	v 	c3h 	F 	   / / /
	 ../../../	 	sCx 	V 	   0 0 0
	38_ c3_f,- 	8 667667667\T  8 8 8Tn %UEu   & $X4 X  %X ,,dkkoo>? $( ==%)
	
 EKK 
 LL	

 U\\"
 
  @
 **+,
 $( ==%)	 EKK 	
 LL U\\"  - 	", #, ../''(<(<= #^^))))) ) 	)
 V) ) )  > 0)X ;;<$$,,44[5I5IJX{#--- - 66>	- $ K =-x HHPPQ
 A #'!AA	A A 	A
 A A E?A 66>A RAH '   ( ))*9  +9 '6 (6 ))488<<89 $( 		D
	D EKK 	D 
&			D
 	D :	D --t||/?/?@A.& .x} . B. 		==>, ?, 		"38 
  #
$ ../ ,0
,, ( \\	 0$ 5: # <A +: 		"  #
 'G (G t{{ + djj ) djj )  - TYY ' TYY ' $.. 1  / ""D$4$4 5  - !!4?? 3 $.. 1 ##T%6%6 7 t{{ + !!4?? 3  - !!4?? 3 djj ) TYY ' t{{ + djj )  - ""D$4$4 5 %%t':': ; TYY 'r,   