
    i                        d dl Z d dlZd dlmZmZmZmZmZmZm	Z	m
Z
mZmZmZ d dlZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2  e
de	      Z3 e
d
e,	      Z4 ejj                  dejl                  z        Z7d ejj                  d      z  Z8d ejj                  dejl                  z        z  Z9 G d d      Z:	 dedededede!deeef   fdZ;dede!dedefdZ<dede=de=de=de=f
dZ>d Z?d Z@d ed!ed"ed#ed$edeeef   fd%ZAd0d&ZBd'e3de3fd(ZCd'e3de3fd)ZDd*e:dedefd+ZEd*e:dedefd,ZFd-ed.efd/ZGy)1    N)AnyIteratorListOptionalSequenceTupleTypeTypeVarUnioncastoverload   )Array1dArray2dArray3dArray4dArrayXd	BatchableDeviceTypesDTypesDTypesFloat	DTypesIntFloats1dFloats2dFloats3dFloats4dFloatsXd	FloatsXdT	GeneratorInts1dInts2dInts3dInts4dIntsXdList2dListXdPaddedShapeSizedGeneratorXp_Floats)get_array_moduleis_xp_arrayto_numpy   )CBlasArrayT)boundFloatsT       @      ?c                   D   e Zd ZU dZeed<   eZeed<   	 dde	de
ddfd	Zdefd
ZdddZddddee
ef   dedede
def
dZddddee
ef   dededede
defdZd Zde
dee
   fdZdddede
dee   defdZddd ede
dee   defd!Z	 	 	 dd"ed#ed$ee   d%ed&edefd'Zd(ed)e
defd*Zd(ed+ed,e defd-Z!e"	 	 	 dd(e#e   d.ee$   d/e
d0e
def
d1       Z%e"	 	 	 dd(e#e   d.ee$   d/e
d0e
def
d2       Z%e"	 	 	 dd(e&d.ee$   d/e
d0e
de'f
d3       Z%e"	 	 	 dd(e(d.ee$   d/e
d0e
de)f
d4       Z%e"	 	 	 dd(e*e)   d.ee$   d/e
d0e
de)f
d5       Z%	 	 	 dd(e*e)   d.ee$   d/e
d0e
de)f
d6Z%e"	 dd(eded/e
de#e   fd7       Z+e"dd(eded/e
de#e   fd8       Z+e"dd(e'ded/e
de&fd9       Z+e"dd(e)ded/e
de(fd:       Z+dd(e)ded/e
de(fd;Z+e"dd<e#e,   de-fd=       Z.e"dd<e#e   de/fd>       Z.	 dd<ee#e,   e#e   f   de0fd?Z.d@e0de#e
   de&fdAZ1d<e&de2fdBZ3d@e2de&fdCZ4dDe5dEee6   de7fdFZ8dGdHdIdJe
d.ee9   dKede fdLZ:dGdHdIdJe
dMe
d.ee9   dKedef
dNZ;dGdHdIdJe
dMe
dOe
d.ee9   dKede/fdPZ<dGdHdIdJe
dMe
dOe
dQe
d.ee9   dKede=fdRZ>dGdHdIdDe5d.ee9   dKede7fdSZ?dTdHdIdJe
d.ee@   dKedefdUZAdTdHdIdJe
dMe
d.ee@   dKede,f
dVZBdTdHdIdJe
dMe
dOe
d.ee@   dKede-fdWZCdTdHdIdJe
dMe
dOe
dQe
d.ee@   dKedeDfdXZEdTdHdIdDe5d.ee@   dKedeFfdYZGdGdHdIdDe5d.ee$   dKedeHfdZZId[e)dJe
deJfd\ZKd[e)dJe
dMe
de'fd]ZLd[e)dJe
dMe
dOe
de0f
d^ZMd[e)dJe
dMe
dOe
dQe
deNfd_ZOd[e7dJe
de fd`ZPd[e7dJe
dMe
defdaZQd[e7dJe
dMe
dOe
de/f
dbZRd[e7dJe
dMe
dOe
dQe
de=fdcZSd[e7dDe5de7fddZTd[eFdJe
defdeZUd[eFdJe
dMe
de,fdfZVd[eFdJe
dMe
dOe
de-f
dgZWd[eFdJe
dMe
dOe
dQe
deDfdhZXd[eFdDe5deFfdiZYd[eZdDe5deZfdjZ[dGdkdlee=e*e*e*e*e6            f   d.ee$   de=fdmZ\dGdkdlee/e*e*e*e6         f   d.ee$   de/fdnZ]dGdkdleee*e*e6      f   d.ee$   defdoZ^dGdkdlee e*e6   f   d.ee$   de fdpZ_dGdkdlee7e*eH   f   d.ee$   de7fdqZ`dTdkdleee*e
   f   d.ee$   defdrZadTdkdlee,e*e*e
      f   d.ee$   de,fdsZbdTdkdlee-e*e*e*e
         f   d.ee$   de-fdtZcdTdkdleeDe*e*e*e*e
            f   d.ee$   deDfduZddTdkdleeFe*eH   f   d.ee$   deFfdvZeddkdlee)e*e)   e*eH   f   d.ee$   de)fdwZfddleZd.ee$   deZfdxZgddyd(ehdzedehfd{Ziddyd ehd|ehdzedehfd}Zjddyd|ehdzedehfd~Zkddyd|eldzedelfdZmddddd"eldzede
de6delf
dZnddddededzede
def
dZodddd|eld elde
de6delf
dZpd ed|ededefdZqde de/de/d(ededereerf   fdZsde de/de/d(ededefdZtd edede derderee f   f
dZud(e/deree,f   fdZvd ede,de
de/fdZwdd(edzedefdZx	 dd ed|edzedefdZy	 	 	 	 	 dd(ehde6de6de6de6dzedehfdZz	 	 	 	 	 dd ehd(ehde6de6de6de6dzedehfdZ{dd(ehde6dzedehfdZ|	 dd ehd(ehde6dzedehf
dZ}dd(ehdzedehfdZ~	 dd ehd(ehdzedehfdZdd(ehdzedehfdZ	 dd ehd(ehdzedehfdZdd(ehdzedehfdZ	 dd ehd(ehd|ehdzedehf
dZdd(ehdzedehfdZ	 dd ehd(ehdzedehfdZdd(ehdzedehfdZ	 dd ehd(ehdzedehfdZdd(ehdzedehfdZ	 dd ehd(ehdzedehfdZd(ehdehfdZd(ehdehfdZdd(ehdzedehfdZ	 dd ehd(ehdzedehfdZdd(ehdzedehfdZ	 dd ehd(ehdzedehfdZ	 dd(ehde6dzedehfdZ	 	 dd ehd(ede6dzedehf
dZ	 ddeldelde
de6ddf
dZ	 dde de de de de6de6de6de6de6dere e e e f   fdĄZdelde6delfdńZdeldelde6fdȄZd(ededefdɄZd(ededereef   fdʄZd(ededereef   fd˄Zd(ededefd̄Zd(edederee,f   fd̈́ZdededefdЄZdededefdӄZdededefdՄZdededefdׄZdede,dedefdلZdede
de,fd܄Zde
dedefdބZ	 dde
de
de
d$ee   def
dZdede,defdZde7deFde7de7fdZd Zy)Opsbasenamexpdevice_type	device_idreturnNc                      || _         || _        y N)r<   r=   )selfr<   r=   kwargss       c/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/thinc/backends/ops.py__init__zOps.__init__@   s     '"    c                 H    t        |       j                   d}t        |      )zReturn C BLAS function table.z" does not provide C BLAS functions)type__name__NotImplementedError)rA   errs     rC   cblasz	Ops.cblasF   s%    d$$%%GH!#&&rE   )
byte_orderc                    t        |t        j                        r6|r2|j                  j	                  |      }t        j
                  ||      }|S t        d      )Ndtypez,Cannot convert non-numpy from base Ops class)
isinstancenumpyndarrayrO   newbyteorderasarray
ValueError)rA   datarL   rO   s       rC   r.   zOps.to_numpyK   sH    dEMM*

//
;}}T7KKLLrE   Fr/   )shufflebuffersizesequencerW   rX   c                R    t        d      sdt               }t        |       j                  t	              t        |t              rt        j                  |      n|      t        j                  t	                     fd}t        |t	                    S )a  Iterate slices from a sequence, optionally shuffled. Slices
        may be either views or copies of the underlying data.

        The `size` argument may be either an integer, or a sequence of integers.
        If a sequence, a new size is drawn before every output.

        If shuffle is True, shuffled batches are produced by first generating
        an index array, shuffling it, and then using it to slice into the
        sequence.

        An internal queue of `buffer` items is accumulated before being each
        output. Buffering is useful for some devices, to allow the
        network to run asynchronously without blocking on every batch.
        __len__z-Can't minibatch data. Expected sequence, got c            
   3   $  K   rt         j                  j                         g } d}D ]S  }t        |      }| j	                  j                  |||z                 t        |       k\  r| E d {    g } ||z  }U | E d {    y 7 7 wNr   rQ   randomrW   intappend
_get_batchlen)	queueirY   rX   indicesrA   rZ   rW   sizess	      rC   _iter_itemsz"Ops.minibatch.<locals>._iter_itemsu   s     $$W-EA 4yT__Xwq1t87LMNu:'$$$ET	  % s$   A0B3B4BBBB)hasattrrG   rU   _get_batch_sizesrd   rP   ra   	itertoolsrepeatrQ   aranger)   )	rA   rY   rZ   rW   rX   rJ   ri   rg   rh   s	   ` ```  @@rC   	minibatchzOps.minibatchT   s    , x+A$x.AQRCS/!%%MZc5J9++D1PT
 ,,s8}-
	 	 k3u:66rE   othersc                   
 |ft        |      z   t        d D              s9dj                  D cg c]  }t        |        c}      }d| }t	        |       j                  t        |      t        |t              rt        j                  |      n|      t        j                  t        |            

 fd}	t        |	t                    S c c}w )zyMinibatch one or more sequences of data, and yield
        lists with one batch per sequence. See ops.minibatch.
        c              3   4   K   | ]  }t        |d         yw)r\   N)rj   .0seqs     rC   	<genexpr>z!Ops.multibatch.<locals>.<genexpr>   s     @s73	*@   z, z/Can't multibatch data. Expected sequences, got c               3   ^  K   	rt         j                  j                         g } d}
D ]p  }t        |      }|||z    }| j	                  g        D ]&  }| d   j	                  j                  ||             ( t        |       k\  r| E d {    g } ||z  }r | E d {    y 7 7 wNr   r;   r_   )re   rf   rY   	idx_batchrZ   rX   rg   rA   	sequencesrW   rh   s        rC   ri   z#Ops.multibatch.<locals>._iter_items   s     $$W-EA 	4y#AD1	R  ) KH"I$$T__Xy%IJKu:'$$$ET		  % s$   BB-B)B-#B+$B-+B-)tuplealljoinrG   rU   rk   rd   rP   ra   rl   rm   rQ   rn   r)   )rA   rY   rZ   rW   rX   rp   ru   valuesrJ   ri   rg   r{   rh   s   `  ``     @@@rC   
multibatchzOps.multibatch   s     K%-/	@i@@YY)D349+DEFCF8LCS/!%%MZc5J9++D1PT
 ,,s8}-	 	" k3u:663  Es   Cc                    t        t              r|D cg c]  }|   	 }}n*t        t              rt        fd|D              }n|   }t        |      r*| j	                  | j
                  j                  |            }|S c c}w )Nc              3   (   K   | ]	  }|     y wr@    )rt   rf   rZ   s     rC   rv   z!Ops._get_batch.<locals>.<genexpr>   s     818A;8s   )rP   listr|   r-   	as_contigr:   rT   )rA   rZ   rg   rf   subseqs    `   rC   rc   zOps._get_batch   su    h%+23ahqk3F3%(888Fg&Fv^^DGGOOF$;<F 4s   Blengthrh   c                 h    g }d}||k  r(|j                  t        |             ||d   z  }||k  r(|S ry   )rb   next)rA   r   rh   outputrf   s        rC   rk   zOps._get_batch_sizes   s@    &jMM$u+&OA &j rE   )lengthsru   nWr   c                `   |dk(  sJ |dk(  sJ |j                   d   }|j                   d   }| j                  ||dz  dz   |      }| j                  |d|  d||      ||dd|f<   ||dd|f<   | j                  ||d d||      |d| |dz   df<   | j                  |||d|z  dz   z        S )a  Given an (M, N) sequence of vectors, return an (M, N*(nW*2+1))
        sequence. The new sequence is constructed by concatenating nW preceding
        and succeeding vectors onto each column in the sequence, to extract a
        window of features.
        r/   Nr   r   r;   )shapealloc3f	reshape3f	reshape2f)rA   ru   r   r   BIcolss          rC   seq2colzOps.seq2col   s     Qww$IIaLIIaL||AQ
Q/DbS	2r1=RS#2#XQU#~~c"#hBBTrcT268^~~dAqAFQJ'788rE   dYc          
         |dk(  sJ |dk(  sJ |dz  dz   }|j                   d   }|j                   d   |z  }| j                  ||      }| j                  ||||      }|d| xxx | j                  ||dd|f   d|      z  ccc ||dd|f   z  }||dxxx | j                  |d| |dz   df   d|      z  ccc |S )zThe reverse/backward operation of the `seq2col` function: calculate
        the gradient of the original `(M, N)` sequence, as a function of the
        gradient of the output `(M, N*(nW*2+1))` sequence.
        r/   Nr   r   r;   )r   alloc2fr   r   )	rA   r   r   r   nFr   r   dXdY3ds	            rC   backprop_seq2colzOps.backprop_seq2col   s     Qww$!VaZHHQKHHQK2\\!Q~~b!R+
4RCDNN4SbS>2q99
d1b5k
234>>$ttR!VX~"6A>>	rE   xyouttrans1trans2c                     |r|j                   }|r|j                   }|| j                  j                  ||      S | j                  j                  |||       |S )zPerform General Matrix Multiplication (GeMM) and optionally store
        the result in the specified output variable.
        r   )Tr:   dot)rA   r   r   r   r   r   s         rC   gemmzOps.gemm   sQ     AA;77;;q!$$GGKK1#K&JrE   Xrepsc                 :    | j                   j                  ||      S r@   )r:   tile)rA   r   r   s      rC   r   zOps.tile   s    ww||At$$rE   Wbc                 8    | j                  ||d      }||z  }|S )zVApply a weights layer and a bias to some inputs, i.e.
        Y = X @ W.T + b
        T)r   )r   )rA   r   r   r   Ys        rC   affinez
Ops.affine  s&     IIa4I(	QrE   rO   padndim_if_emptyc                      y r@   r   rA   r   rO   r   r   s        rC   flattenzOps.flatten  s     rE   c                      y r@   r   r   s        rC   r   zOps.flatten  s     rE   c                      y r@   r   r   s        rC   r   zOps.flatten       rE   c                      y r@   r   r   s        rC   r   zOps.flatten(  r   rE   c                      y r@   r   r   s        rC   r   zOps.flatten1  r   rE   c           	         |t        |      dk(  r| j                  d|z  |xs d      S t        |d         }|d   j                  }|D cg c]  }|j                  dk7  s| }}t        |      dk(  r| j                  ||xs d      S t        |      dk\  rg }|D ]P  }|j                  |j                  |f|j                  dd z   |j                               |j                  |       R |j                  |j                  |fj                  dd z   |j                               |}|j                  |      }	||j                  |	|      }	|	S c c}w )z.Flatten a list of arrays into one large array.Nr   r   frN   r/   )rd   allocr,   r   rY   ra   rb   zerosrO   concatenaterT   )
rA   r   rO   r   r   r:   shape_if_emptyr   paddedresults
             rC   r   zOps.flatten:  sG    9A!::d]2%,3:GGad#1)1QVVq[Q))q6Q;::nELS:AAs8q=F !bhhv';177hKLa ! MM"((C6AGGABK#7qww(GHA"ZZeZ4F *s   E!Ec                      y r@   r   rA   r   r   r   s       rC   	unflattenzOps.unflattenU  s     rE   c                      y r@   r   r   s       rC   r   zOps.unflattenZ  s    SVrE   c                      y r@   r   r   s       rC   r   zOps.unflatten]      NQrE   c                      y r@   r   r   s       rC   r   zOps.unflattenb  r   rE   c                 .   t        |      }|dkD  rt        j                  |dkD  ||z   d      }| j                  j	                  |t        j
                  |            dd }|dkD  r|D cg c]  }||d 	 }}t        |      t        |      k(  sJ |S c c}w )zThe reverse/backward operation of the `flatten` function: unflatten
        a large array into a list of arrays according to the given lengths.
        r   Nr;   )r.   rQ   wherer:   splitcumsumrd   )rA   r   r   r   unflatas         rC   r   zOps.unflattene  s    
 7#7kk'A+w}a@Gq%,,w"78"=7'-.!ag.F.6{c'l***	 /s   )Bseqsc                      y r@   r   rA   r   round_tos      rC   r   zOps.padv  s    =@rE   c                      y r@   r   r   s      rC   r   zOps.pady  s    ADrE   c                 `   |dk  rt        d|       |st        d      t        t        d |D                    dk7  rt        d      t        t        d |D                    dk7  rt        d      t        t        d |D                    dk7  rt        d	      t        d
 |D              }|| |z  z  }t        |      |f|d   j                  dd z   }t        t        | j                  ||d   j                              }t        |      D ]  \  }}|||d|j                  d   f<    |S )zPerform padding on a list of arrays so that they each have the same
        length, by taking the maximum dimension across each axis. This only
        works on non-empty sequences with the same `ndim` and `dtype`.
        r/   z.Rounding for padding must at least be 1, was: zCannot pad empty sequencec              3   4   K   | ]  }|j                     y wr@   )ndimrs   s     rC   rv   zOps.pad.<locals>.<genexpr>  s     ,388,rw   z)Cannot pad sequences with different ndimsc              3   4   K   | ]  }|j                     y wr@   rN   rs   s     rC   rv   zOps.pad.<locals>.<genexpr>  s     -399-rw   z*Cannot pad sequences with different dtypesc              3   :   K   | ]  }|j                   d d   yw)r/   N)r   rs   s     rC   rv   zOps.pad.<locals>.<genexpr>  s     1S399QR=1s   z4Cannot pad sequences that differ on other dimensionsc              3   2   K   | ]  }t        |        y wr@   )rd   rs   s     rC   rv   zOps.pad.<locals>.<genexpr>  s     3s#c(3s   r   NrN   )
rU   rd   setmaxr   r   r   r   rO   	enumerate)rA   r   r   max_seq_lenfinal_shaper   rf   arrs           rC   r   zOps.pad|  s>    a<@
K 
 899s,t,,-2HIIs---.!3IJJs1D112a7STT3d33 	|h..4y+.aqr1BBw

;d1gmm
(TUo 	,FAs(+F1n		!n$%	, rE   r   c                 |    g }t        |      D ]  \  }}|j                  ||d|f           t        t        |      S )zThe reverse/backward operation of the `pad` function: transform an
        array back into a list of arrays, each with their original length.
        N)r   rb   r   r%   )rA   r   r   r   rf   r   s         rC   unpadz	Ops.unpad  sG     "7+ 	.IAvMM&GVG,-	.FF##rE   c                    |sLt        | j                  ddd      | j                  d      | j                  d      | j                  d            S t        |      dk(  r| j	                  |d   |d   j
                  d   d|d   j
                  d         }| j                  dg|j
                  d   z        }| j                  |j
                  d   g      }| j                  dg      }t        ||||      S t        |      D cg c]  \  }}t        |      |f }}}|j                  d       |D 	cg c]  \  }	}|	 }
}	}|D 	cg c]  \  }	}|		 }}	}t        |D cg c]  }|j
                  d    c}      }t        |      }|d   j
                  d   }t        t        |
D cg c]  }||   	 c}      }| j                  |      }|j
                  |||fk(  s
J |||f       | j                  |j                  d            }|j
                  |||fk(  sJ t        |      D cg c]  }d }}t        |      }t        |      D ]'  }|r|||dz
     k\  r|dz  }|r|||dz
     k\  r|||<   ) t!        |      t!        |      k(  sJ t        || j                  |      | j                  |      | j                  |
            S c c}}w c c}}	w c c}}	w c c}w c c}w c c}w )z4Pack a sequence of 2d arrays into a Padded datatype.r   r/   T)reverser/   r   r   )r'   r   alloc1ird   reshape3r   	asarray1ir   sortr   r   r%   r   r   	transposerangesum)rA   r   rV   	size_at_tr   rg   rf   ru   lengths_indicesr   indices_lengths_nSnBnOr   _batch_size_at_t_current_sizets                       rC   list2paddedzOps.list2padded  s   Q1%t||AQVW  Y!^==a$q'--*:AtAw}}Q?OPDsTZZ]':;Inndjjm_5GnnaS)G$	7G<<7@GVQCHa=GGT*'67)&!A77,;<yvqF<<$/3#))A,/0Y!W]]1 Fh7T!W78xx~yyRRL(62r2,6(nnS]]956yyRRL(((',Ry1!A118}r 	/A11A(B#B! 11A(B#B".Q	/ 8}$4 5555NN+,NN8$NN8$	
 	
/ H7</ 8 2s$   K4KK K(K
)	K$c                 \   |j                   }t        |j                        }t        |j                        }dgt	        |      z  }| j                  |j                  d            }t        |j                  d         D ]  }||dt        ||         f   |||   <    t        t        |      S )z;Unpack a Padded datatype to a list of 2-dimensional arrays.Nr   r   )rV   r.   rg   r   rd   r   r   r   r   ra   r   r%   )rA   r   rV   rg   r   unpaddedrf   s          rC   padded2listzOps.padded2list  s    {{6>>*6>>*-1FS\,A~~dnnY78tzz!}% 	>A#'+<S_+<(<#=HWQZ 	>FH%%rE   r   dropc                    ||dk  r| j                   j                  |d      S |dk\  r| j                  |      S | j                   j                  j	                  dd|      }||k\  d|z
  z  }t        t        | j                  |d            S )a  Create a random mask for applying dropout, with a certain percent of
        the mask (defined by `drop`) will contain zeros. The neurons at those
        positions will be deactivated during training, resulting in a more
        robust network and less overfitting.
        r   r   rN   r5           float32)r:   onesalloc_fr`   uniformr   r   rT   )rA   r   r   	coinflipsmasks        rC   get_dropout_maskzOps.get_dropout_mask  s     <41977<<S<11S[<<&&GGNN**3U;	T!cDj1Hdll4ylABBrE   r   TrO   r   d0r   c                H    t        t        | j                  |f||            S Nr  )r   r   r   rA   r  rO   r   s       rC   alloc1fzOps.alloc1f  s"     Hdjj"e5jIJJrE   d1c                J    t        t        | j                  ||f||            S r	  )r   r   r   rA   r  r  rO   r   s        rC   r   zOps.alloc2f  s$     Hdjj"bejLMMrE   d2c                L    t        t        | j                  |||f||            S r	  )r   r   r   rA   r  r  r  rO   r   s         rC   r   zOps.alloc3f  s&     Hdjj"b"U%jPQQrE   d3c          	      N    t        t        | j                  ||||f||            S r	  )r   r   r   rA   r  r  r  r  rO   r   s          rC   alloc4fzOps.alloc4f  s)     Hdjj"b"b)9ejTUUrE   c                F    t        t        | j                  |||            S r	  )r   r   r   rA   r   rO   r   s       rC   r  zOps.alloc_f  s      Hdjje5jIJJrE   int32c                H    t        t        | j                  |f||            S r	  )r   r    r   r
  s       rC   r   zOps.alloc1i  s"     FDJJuEJGHHrE   c                J    t        t        | j                  ||f||            S r	  )r   r!   r   r  s        rC   alloc2izOps.alloc2i&  s$     FDJJBxuEJJKKrE   c                L    t        t        | j                  |||f||            S r	  )r   r"   r   r  s         rC   alloc3izOps.alloc3i0  s&     FDJJB|5JNOOrE   c          	      N    t        t        | j                  ||||f||            S r	  )r   r#   r   r  s          rC   alloc4izOps.alloc4i;  s)     FDJJBB'7uEJRSSrE   c                F    t        t        | j                  |||            S r	  )r   r$   r   r  s       rC   alloc_izOps.alloc_iG  s      FDJJuEJGHHrE   c                    t        |t              r|f}|r| j                  j                  ||      S | j                  j	                  ||      S )z%Allocate an array of a certain shape.rN   )rP   ra   r:   r   emptyr  s       rC   r   z	Ops.allocP  sF     eS!HE77==e=4477==e=44rE   arrayc                 D    t        t        | j                  ||f            S r@   )r   r   reshaperA   r$  r  s      rC   reshape1zOps.reshape1`  s    GT\\%"788rE   c                 F    t        t        | j                  |||f            S r@   )r   r   r&  rA   r$  r  r  s       rC   reshape2zOps.reshape2c  s    GT\\%"b:;;rE   c           	      H    t        t        | j                  ||||f            S r@   )r   r   r&  rA   r$  r  r  r  s        rC   r   zOps.reshape3f  s     GT\\%"b">??rE   c           
      J    t        t        | j                  |||||f            S r@   )r   r   r&  rA   r$  r  r  r  r  s         rC   reshape4zOps.reshape4i  s#    GT\\%"b"b1ABCCrE   c                 D    t        t        | j                  ||f            S r@   )r   r   r&  r'  s      rC   	reshape1fzOps.reshape1fl  s    Hdll52%899rE   c                 F    t        t        | j                  |||f            S r@   )r   r   r&  r*  s       rC   r   zOps.reshape2fo  s    Hdll52r(;<<rE   c           	      H    t        t        | j                  ||||f            S r@   )r   r   r&  r-  s        rC   r   zOps.reshape3fr  s     Hdll52r2,?@@rE   c           
      J    t        t        | j                  |||||f            S r@   )r   r   r&  r/  s         rC   	reshape4fzOps.reshape4fu  s%     Hdll52r2r2BCDDrE   c                 &    | j                  ||      S r@   r&  rA   r$  r   s      rC   	reshape_fzOps.reshape_fz      ||E5))rE   c                 D    t        t        | j                  ||f            S r@   )r   r    r&  r'  s      rC   	reshape1izOps.reshape1i}  s    FDLL677rE   c                 F    t        t        | j                  |||f            S r@   )r   r!   r&  r*  s       rC   	reshape2izOps.reshape2i  s    FDLLR9::rE   c           	      H    t        t        | j                  ||||f            S r@   )r   r"   r&  r-  s        rC   	reshape3izOps.reshape3i  s     FDLLR=>>rE   c           
      J    t        t        | j                  |||||f            S r@   )r   r#   r&  r/  s         rC   	reshape4izOps.reshape4i  s#    FDLLRR0@ABBrE   c                 &    | j                  ||      S r@   r8  r9  s      rC   	reshape_izOps.reshape_i  r;  rE   c                 f    t        |t              r|f}t        t        |j	                  |            S )zReshape an array.)rP   ra   r   r1   r&  r9  s      rC   r&  zOps.reshape  s)    eS!HEFEMM%011rE   rN   rV   c                D    t        t        | j                  ||            S NrN   )r   r   rT   rA   rV   rO   s      rC   	asarray4fzOps.asarray4f       Hdll4ul=>>rE   c                D    t        t        | j                  ||            S rH  )r   r   rT   rI  s      rC   	asarray3fzOps.asarray3f  rK  rE   c                D    t        t        | j                  ||            S rH  )r   r   rT   rI  s      rC   	asarray2fzOps.asarray2f  rK  rE   c                D    t        t        | j                  ||            S rH  )r   r   rT   rI  s      rC   	asarray1fzOps.asarray1f  rK  rE   c                D    t        t        | j                  ||            S rH  )r   r   rT   rI  s      rC   	asarray_fzOps.asarray_f  rK  rE   c                D    t        t        | j                  ||            S rH  )r   r    rT   rI  s      rC   r   zOps.asarray1i       FDLLUL;<<rE   c                D    t        t        | j                  ||            S rH  )r   r!   rT   rI  s      rC   	asarray2izOps.asarray2i       FDLLUL;<<rE   c                D    t        t        | j                  ||            S rH  )r   r"   rT   rI  s      rC   	asarray3izOps.asarray3i  rX  rE   c                D    t        t        | j                  ||            S rH  )r   r#   rT   rI  s      rC   	asarray4izOps.asarray4i  rX  rE   c                D    t        t        | j                  ||            S rH  )r   r$   rT   rI  s      rC   	asarray_izOps.asarray_i  rU  rE   c                R   t        || j                  j                        r2||S |j                  |k(  r|S | j                  j	                  ||      S t        |d      r|j                         S || j                  j                  ||      S | j                  j                  |      S )z,Ensure a given array is of the correct type.rN   rQ   )rP   r:   rR   rO   rT   rj   rQ   r$  rI  s      rC   rT   zOps.asarray  s     dDGGOO,}u$wwt599T7#::<77==U=3377==&&rE   c                     |j                   d   r|d|j                  fv r|S |d|ini } | j                  j                  |fi |S )zAllow the backend to make a contiguous copy of an array.
        Implementations of `Ops` do not have to make a copy or make it
        contiguous if that would not improve efficiency for the execution engine.
        C_CONTIGUOUSNrO   )flagsrO   r:   ascontiguousarray)rA   rV   rO   rB   s       rC   r   zOps.as_contig  sR    
 ::n%%D$**3E*EK%*%6'5!B(tww((888rE   inplacere  c                   |rI| j                   j                  |dd|      }| j                   j                  | |       |dz  }|dz  }|S | j                   j                  |dd      }dd| j                   j                  |       z   z  S )N      4      4@r   r5         )r:   clipexprA   r   re  s      rC   sigmoidzOps.sigmoid  s    Qt3AGGKKK"HA$JAHQt,A#QB/00rE   r   c                f    |r| j                  |d       ||z  }|S || j                  ||      z  S )NTrd  )dsigmoidrA   r   r   re  s       rC   backprop_sigmoidzOps.backprop_sigmoid  s=     MM!TM*GAHa999rE   c                *    |r
|d|z
  z  }|S |d|z
  z  S )Nr/   r5   r   rA   r   re  s      rC   ro  zOps.dsigmoid  s%    QJAHa= rE   c                8    |r|dz  }|dz  }|dz  }|S d|dz  z
  S )Nr   ri  r5   r/   r   rs  s      rC   dtanhz	Ops.dtanh  s2    !GAIAHAHq!t8OrE   r5   )re  axistemperaturerv  rw  c                    |dk7  r||z  }| j                   j                  ||d      }||z
  }| j                   j                  |      }||j                  |d      z  }|S Nr5   T)rv  keepdims)r:   r   rk  r   )rA   r   re  rv  rw  maxesshiftednew_xs           rC   softmaxzOps.softmax"  sd     #KAAD48e)G$t44rE   )re  rv  Xsc                   |j                   dk\  rd|j                    }t        |      | j                  j                  |dd      }| j                  j	                  |      }| j                  | j                  ||      |      }||z  }|S )N   z)Softmax currently only supports 2d. Got: rg  rh  )r   rI   r:   rj  rk  backprop_reduce_sum
reduce_sum)rA   r  r   re  rv  rJ   r}  summeds           rC   softmax_sequenceszOps.softmax_sequences2  s|     77a<=bggYGC%c**WW\\"eT*B))$//%*I7SrE   )rv  rw  c                V    |dk7  r||z  }||z  }|||j                  |d      z  z  }|S ry  r   )rA   r   r   rv  rw  r   s         rC   backprop_softmaxzOps.backprop_softmax?  s@     #k!BV
a"&&dT&222	rE   c                 d    ||z  }| j                  | j                  ||      |      }|||z  z  }|S r@   )r  r  )rA   r   r   r   r   sum_dXs         rC   backprop_softmax_sequenceszOps.backprop_softmax_sequencesI  s=     V))$//"g*FP
a&j	rE   paramsH0C0r   c                     |j                   |j                   k(  sJ |j                   d   |j                   d   k(  sJ t        |||||      \  }}||fS Nr/   )r   lstm_forward_training)rA   r  r  r  r   r   r   	fwd_states           rC   r  zOps.lstm_forward_trainingQ  sX     xx288###xx{bhhqk))),VRQ	J9)|rE   c                 *    t        |||||      \  }}|S r@   )r  )rA   r  r  r  r   r   r   r   s           rC   lstm_forward_inferencezOps.lstm_forward_inference^  s     %VRQ	B1rE   r  c                 ,    t        ||||      \  }}||fS r@   )backprop_lstm)rA   r   r   r  r  r   d_paramss          rC   r  zOps.backprop_lstmi  s"     %R&)DH8|rE   c                 N    |j                  d      }|j                  d      |fS )Nr;   rv  )argmaxr   )rA   r   whichs      rC   maxoutz
Ops.maxouto  s'    b!uu"u~u$$rE   r  Pc           	         | j                  |j                  d   |j                  d   ||j                        }t        |j                  d         D ]1  }t        |j                  d         D ]  }|||f   ||||||f   f<    3 |S )Nr   r/   rN   )r   r   rO   r   )rA   r   r  r  r   r   os          rC   backprop_maxoutzOps.backprop_maxouts  s    \\"((1+rxx{ARXX\Frxx{# 	1A288A;' 1(*1a41aq!t$%1	1 	rE   c                 *    |s||dkD  z  S ||dkD  z  }|S r^   r   rl  s      rC   reluzOps.reluz  s%    A;QJAHrE   c                 *    |s||dkD  z  S ||dkD  z  }|S r^   r   rp  s       rC   backprop_reluzOps.backprop_relu  s'     Q<
a!e	rE   slopeoffsetmin_valmax_valc                     |r)||z  }||z  }| j                   j                  ||||      S ||z  |z   }| j                   j                  |||      S )Nr   )r:   rj  )rA   r   r  r  r  r  re  r   s           rC   clipped_linearzOps.clipped_linear  sY     JAKA77<<7G<;;%i& ww||C'22rE   c                 Z   ||z
  |z  }||z
  |z  }	| j                   j                  |      j                  |j                        }| j                   j                  d      j                  |j                        }
| j                   j	                  ||k  ||	k  z  ||
      }|r||z  }|S ||z  S )Nr   )r:   float64astyperO   r   )rA   r   r   r  r  r  r  re  lowhighzeror   s               rC   backprop_clipped_linearzOps.backprop_clipped_linear  s     5(& E)&--agg6wws#**1773WW]]C!GD15$?"HBIBwrE   nc                 *    | j                  |||      S N)r  re  r  )rA   r   r  re  s       rC   relu_kz
Ops.relu_k  s    ""1a"AArE   c                 ,    | j                  ||||      S r  r  )rA   r   r   r  re  s        rC   backprop_relu_kzOps.backprop_relu_k  s     ++B1g+NNrE   c                 ,    | j                  |dd|      S )N皙?      ?)r  r  re  r  rl  s      rC   hard_sigmoidzOps.hard_sigmoid  s    ""1CW"MMrE   c                 ,    | j                  ||dd      S )Nr  r  )r  r  r  rA   r   r   re  s       rC   backprop_hard_sigmoidzOps.backprop_hard_sigmoid  s     ++BS+IIrE   c                 ,    | j                  |dd|      S )Nri  r5   )r  r  re  r  rl  s      rC   	hard_tanhzOps.hard_tanh  s    ""1dC"QQrE   c                 ,    | j                  ||dd      S )Nri  r5   )r  r  r  r  s       rC   backprop_hard_tanhzOps.backprop_hard_tanh  s     ++B4+MMrE   c                 ^    |r|| j                  |      z  }|S || j                  |      z  }|S r@   rm  rA   r   re  r   s       rC   swishz	Ops.swish  s3    a AH$,,q/!
rE   c                 V    || j                  |      d|z
  z  z   }|r||z  }|S ||z  }|S r  r  )rA   r   r   r   re  r   s         rC   backprop_swishzOps.backprop_swish  s>     Q1q5))!GBI1f
rE   c                 ^    |r|| j                  |      z  }|S || j                  |      z  }|S r@   )r  r  s       rC   
hard_swishzOps.hard_swish  s9    ""1%%AH$##A&&
rE   c                 N    |dz  dz   }d||dkD  <   d||dk  <   |r||z  }|S ||z  S )Ng?r  r5   g      @r   g      r   rA   r   r   re  r   s        rC   backprop_hard_swishzOps.backprop_hard_swish  sE     Ws]1s71t8"HBIBwrE   c                 r    |r|| j                  |dz         dz  z  }|S || j                  |dz         dz  z  S )Nr     )r  rl  s      rC   hard_swish_mobilenetzOps.hard_swish_mobilenet  sC    QU#a''AHDKKA&*++rE   c                 T    d|dz  dz   z  }d||dkD  <   d||dk  <   |r||z  }|S ||z  S )NgUUUUUU?r4         @r5   r   g      r   r  s        rC   backprop_hard_swish_mobilenetz!Ops.backprop_hard_swish_mobilenet  sJ     C#&1s71t8"HBIBwrE   c                     | j                   j                  |      }|dz  }| j                   j                  ||       ||z  }|dz  }|dz  }|r||z  }|S ||z  S )Nr5   r   r/   r  r:   squaresqrt)rA   r   re  tmps       rC   dishzOps.dish  sf    ggnnQs
Sc"#gqs
HAHs7NrE   c                     | j                   j                  |      }|dz   }|| j                   j                  |      z  }d|z  |z  }||dz  z  }||z  }|dz  }|r||z  }|S ||z  S )Nr5   r  g      ?r  )rA   r   r   re  x_sqx_sq_plus_onederivseconds           rC   backprop_dishzOps.backprop_dish  s     ww~~a s
DGGLL//q4-$$%KBI:rE   c                 `   | j                   j                  |      }| j                   j                  |      }d}d}d}d}d}d}dd||z  z   z  }	d||	z  |z   |	z  |z   |	z  |z   |	z  |z   |	z  | j                   j                  | |z        z  z
  }
||
z  }|j	                  |j
                        }|S )Ng~Z O?gi<15ҿgWU?g9LW@g-UB?g{=@?r5   )r:   signabsrk  r  rO   )rA   r   r  a1a2a3a4a5pr   r   r   s               rC   erfzOps.erf  s    ww||AGGKKN3Q;rAv{a'2-2R71<rAQFBFJ
 
 
 Qhjj!
rE   c                 ~    | j                   j                  |dd      }d| j                   j                  |      z  dz  S )Nrg  rh  r/   r   )r:   rj  cosh)rA   r   s     rC   sechsqz
Ops.sechsq.  s4    GGLLE4(DGGLLO#))rE   c           
      "   d| j                   j                  t        |d| j                   j                  |d      z  z   z        z   }|dz  }|j	                  |j
                        }|r||z  }|S | j                   j                  |      }||z  }|S )Nr5   gHm?r  r  )r:   tanhSQRT2PIpowerr  rO   r$  )rA   r   re  r  r   s        rC   gelu_approxzOps.gelu_approx3  s    DGGLLA477==A;N0N,N!OPPs
jj!HAHa AHAHrE   c                 \   t        t        | j                  |j                              }| j                  j                  |d      }d| j                  j                  d|z  d|z  z         z  }|d|z  d|z  z   | j                  d|z  d|z  z         z  z  }|dz  }||z  }|r||z  }|S ||z  S )Nr  r  giND?gF?gFrVvf?gD?)r   r   r  r   r:   r  r  r  )rA   r   r   re  r   Xp3r  s          rC   backprop_gelu_approxzOps.backprop_gelu_approx?  s     )T\\!''23ggmmAq!DGGLLS8a<!?@@	C(Q,.$++Ohl*3
 
 	
 	s

c	"HBIBwrE   c                 6    t        | |      }|r||z  }|S ||z  S r@   )gaussian_cdf)rA   r   re  cdfs       rC   geluzOps.geluO  s(    4#HAH3wrE   c                 V    t        | |      |t        | |      z  z   }|r||z  }|S ||z  S r@   )r  gaussian_pdfr  s        rC   backprop_geluzOps.backprop_geluW  s;     $"QdA)>%>>"HBIBwrE   	thresholdc           	          || j                   j                  | j                   j                  d| j                   j                  |      z               z  }| j                   j	                  ||k\  ||      }|r||d d  |S |S Nr5   )r:   r  logrk  r   )rA   r   r  re  r  r   s         rC   mishzOps.misha  si     $'',,tww{{3Q+?@AAGGMM!y.!S1AaDHHrE   c                    |j                   |j                   k7  r'd|j                    d|j                    }t        |      t        |      }||k  }||   }||   }	d|dz   z  }
|
d|j                  d|z        z  z  }
|
|j                  d|z        z  }
|
|j                  |      d|z  dz   z  z  }
|j                  |      dz   }||z  }|dz  }|	|j                  |      |
z  |dz  z  z  }|r|}n|j	                  |      }|||<   |S )	N!arrays have incompatible shapes:  and g      @r5   r4   r        @r   )r   rU   r,   rk  copy)rA   r   r   r  re  msgr:   rg   XsubdYsubomegadeltadXsubr   s                 rC   backprop_mishzOps.backprop_mishl  s%    88qww5bhhZuQWWINCS/!a i-z7tcz"rvvcDj)))d
###*!344ts""&&,.5!8<=C''"+CG
rE   emaweightsr   	max_decayc                 D    d|z   d|z   z  }||kD  r|}|d|z
  ||z
  z  z  }y )Nr5   g      $@r/   r   )rA   r  r  r   r  decays         rC   update_averageszOps.update_averages  s:     qTAX&9EE	cGm,,rE   gradientmom1mom2beta1beta2eps
learn_ratemod_ratec
                     t        ||       t        ||       t        ||       ||z  }||z  }||d|z
  z  z  }|||z  d|z
  z  z  }||||	| j                  j                  |      z  |z   z  z  z  }||||fS r   )_check_compatible_shaper:   r  )
rA   r  r  r  r  r  r  r  r  r  s
             rC   adamzOps.adam  s     	 2.. 	C%K((8#sU{33 	:DGGLL4F)F)L!MNN$,,rE   c                 l    t        |      }|j                  j                  |      }||k\  r|||z  z  }|S r@   )r,   linalgnorm)rA   r  r  r:   	grad_norms        rC   clip_gradientzOps.clip_gradient  s:    h'IINN8,		!	I--HrE   y_truey_predc                     | j                   j                  |dz         }||z  d|z
  | j                   j                  d|z
  dz         z  z   }| S )Ng:0yE>r/   )r:   r  )rA   r%  r&  log_yplosss        rC   loglosszOps.logloss  sL    Vd]+AJ$''++q6zT>Q2R#RRurE   c                 B   | j                  |j                  d   |j                  d   d      }d}t        |      D ]`  \  }}|dk  rt        d|       ||z   |j                  d   kD  rt	        d      |r!||||z    j                  d      ||<   ||z  }\d||<   b |S 	Nr   r/   Fr   'all sequence lengths must be >= 0, got )lengths must sum up to the number of rowsr  r   )r   r   r   rU   
IndexErrorr   rA   r   r   r   startrf   r   s          rC   r  zOps.reduce_sum  s    LLq)1771:ULC"7+ 		IAvz #J6(!STT!''!*, !LMM044!4<!!		 rE   c                 ~   |j                   dk(  r!| j                  d|j                  d         |fS | j                  j	                  |dkD        st        d      | j                  |j                  d   dz   d      }d|d<   |j                         |dd  |d   |j                  d   k7  rt        d      ||d d    |fS )Nr   r/    all sequence lengths must be > 0Fr-  r;   r/  )	rY   r   r   r:   r}   rU   r   r   r0  )rA   r   r   starts_endss       rC   reduce_firstzOps.reduce_first  s    <<1<<1771:.77ww{{7Q;'?AAll7==#3a#7ulEA!..*ABr?aggaj(HIISb!"K//rE   c                 .   |j                   dk(  r!| j                  d|j                  d         |fS | j                  j	                  |dkD        st        d      |j                         dz
  }|d   dz   |j                  d   k7  rt        d      ||   |fS )Nr   r/   r4  r;   r/  )rY   r   r   r:   r}   rU   r   r0  )rA   r   r   lastss       rC   reduce_lastzOps.reduce_last  s    <<1<<1771:.77ww{{7Q;'?AA 1$9q=AGGAJ&HIIxrE   c                 B   | j                  |j                  d   |j                  d   d      }d}t        |      D ]`  \  }}|dk  rt        d|       ||z   |j                  d   kD  rt	        d      |r||||z    j                  d      ||<   nd||<   ||z  }b |S r,  )r   r   r   rU   r0  meanr1  s          rC   reduce_meanzOps.reduce_mean  s    LLq)1771:ULC"7+ 		IAvz #J6(!STT!''!*, !LMM05515=!!VOE		 rE   c                    | j                  |j                  d   |j                  d   |j                  d      }| j                  |j                  d   |j                  d   d      }d}t	        |      D ]u  \  }}|dk  rt        d|       ||z   |j                  d   kD  rt        d      |r6||||z    j                  d      ||<   ||||z    j                  d      ||<   ||z  }w ||fS )	Nr   r/   Fr  r-  &all sequence lengths must be > 0, got r/  r  )	r   r   rO   r  r   rU   r0  r  r   )rA   r   r   r   r  r2  rf   r   s           rC   
reduce_maxzOps.reduce_max  s   LLq)1771:QWWELRW]]1-qwwqzG"7+ 	IAv{ #I&!RSS!''!*, !LMMUUV^4;;;Ca044!4<!VOE	 %xrE   d_firstsr5  c                 0   |j                   dk(  r,| j                  d|j                  d   |j                  d      S |j                   dk(  rt	        d      | j                  t        |d         |j                  d   |j                  d      }|||d d <   |S )Nr   r/   Tr  z starts_ends must not have size 1r;   )rY   r   r   rO   rU   ra   )rA   r@  r5  r   s       rC   backprop_reduce_firstzOps.backprop_reduce_first   s     q <<8>>!#4HNNRV<WW"?AA\\B (.."38>>QU  
  (;s	rE   d_lastsr8  c                     |j                   dk(  r,| j                  d|j                  d   |j                  d      S | j                  t	        |d         dz   |j                  d   |j                  d      }|||<   |S )Nr   r/   Tr  r;   )rY   r   r   rO   ra   )rA   rC  r8  r   s       rC   backprop_reduce_lastzOps.backprop_reduce_last  sx    ::?<<7==#37==PT<UU\\b	NQa 0T  
 5		rE   d_sumsc                     | j                  |j                         |j                  d   |j                  d      }d}t	        |      D ](  \  }}|dk  rt        d|       ||   ||||z    ||z  }* |S Nr/   Fr  r   r.  r   r   r   rO   r   rU   )rA   rF  r   r   r2  rf   r   s          rC   r  zOps.backprop_reduce_sum  s    \\KKM6<<?&,,e  
 "7+ 	IAvz #J6(!STT)/Buuv~&VOE		
 	rE   d_meansc                     | j                  |j                         |j                  d   |j                  d      }d}t	        |      D ]+  \  }}|dk  rt        d|       ||   |z  ||||z    ||z  }- |S rH  rI  )rA   rJ  r   r   r2  rf   r   s          rC   backprop_reduce_meanzOps.backprop_reduce_mean"  s    \\KKM7==+7==  
 "7+ 	IAvz #J6(!STT)0f)<Buuv~&VOE		
 	rE   d_maxesc                 D   | j                  |j                         |j                  d   |j                        }d}t	        |      D ]V  \  }}|dk  rt        d|       | j                  j                  ||||z    ||   j                  d      ||   d       ||z  }X |S )Nr/   rN   r   r>  )r/   r;   )	r   r   r   rO   r   rU   r:   put_along_axisr&  )rA   rM  r  r   r   r2  rf   r   s           rC   backprop_reduce_maxzOps.backprop_reduce_max.  s     \\'++-q)9\O"7+ 	IAv{ #I&!RSSGG""556>*E!H,<,<W,EwqzST VOE	 	rE   idsseedc                     ddl m}  |       }| j                  |j                  |j	                  |d      |            S )zcHash a sequence of 64-bit keys into a table with 4 32-bit keys, using
        murmurhash3.
        r/   NumpyOpsuint64rN   )	numpy_opsrU  rW  hashrT   )rA   rQ  rR  rU  rW  s        rC   rX  zOps.hash=  s=     	(J	~~NN9,,S,A4H
 	
rE   keysc           	          ddl m}  |       }| j                  |j                  ||j	                  |d                  S )Nr/   rT  rV  rN   )rW  rU  r   ngramsrT   )rA   r  rY  rU  rW  s        rC   r[  z
Ops.ngramsH  s=    'J	~~Q	 1 1$h 1 GH
 	
rE   NDperiodc                 b    ddl m}  |       }| j                  |j                  ||||            S )Nr/   rT  )rW  rU  rO  position_encode)rA   r\  r]  r^  r   rU  rW  s          rC   r`  zOps.position_encodeP  s.     	(J	~~i771fcJKKrE   tablerg   c                 ,    ||   j                  d      S )Nr/   r  r  )rA   ra  rg   s      rC   
gather_addzOps.gather_addY  s    W~!!q!))rE   r   c                 P    | j                   j                  j                  |||      S r@   )r:   addat)rA   ra  rg   r   s       rC   scatter_addzOps.scatter_add\  s     ww{{~~eWf55rE   c                     | j                  ||d   j                        }t        |      D ]  \  }}|||d|j                  d   f<    |S )z8Maybe don't need this? Just a quicky to get Jax working.r   rN   N)r   rO   r   r   )rA   r   r  r   rf   r   s         rC   insert_intozOps.insert_intoa  sR    EA5bM 	(DAq&'F1l
l?#	(rE   )cpur;   )NFF)Nr   r   r   )r/   r@   )F)r5   r   r   r5   F)r  F)rh  F)gH.?)r5   )i'  N)rH   
__module____qualname__r9   str__annotations__rQ   r:   r*   r   ra   rD   r0   rK   r.   r   r   r   boolr)   ro   r   rc   r   rk   r   r   r    r   r   r   r   r   r   r   r   r   r   r%   r   r&   r   r   r   r!   r"   r   r   r   r   r'   r   r   r(   floatr   r  r   r  r   r   r   r  r  r   r   r  r  r#   r  r$   r!  r   r   r   r(  r+  r   r   r0  r2  r   r   r6  r:  r=  r?  rA  rC  rE  r1   r&  rJ  rM  rO  rQ  rS  r   rW  rZ  r\  r^  rT   r   r   rm  rq  ro  r3   ru  r~  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r$  r*  r  r6  r9  r<  r?  rB  rE  r  rL  rP  rX  r[  r`  rc  rg  ri  r   rE   rC   r7   r7   <   s   D#BN BD#&#;>#	#'u '
 ,0 M /7CN#/7 /7
 /7 /7 
/7l (7CN#(7 (7 	(7
 (7 (7 
(7T	s 8C=  FJ99!$92:62B9	9* EI #19&1A	2 #'  h	
   
*%h %c %h % X ( x   #'>  	
  
   #'<  	
  
   #'  	
  
   #'  	
  
   #'G  	
  
  #'G  	
  
6 78$*14	h  V6VFVVT&\V VQ7QVQ#QfQ Q Q7QVQ#QfQ Q7 V # f " @V@V@ @DXDxD D CD$v,X67	B$G $d3i $F $(
 (
6 (
T
&& 
&V 
&Ce C8E? Cx C$ (1KK $	K
 K 
K (1NN N
 $N N 
N  (1	R	R 	R 		R $	R 	R 
	R$ (1
V
V 
V 	
V
 
V $
V 
V 

V  (1KK $	K
 K 
K &-II 	"	I
 I 
I &-LL L
 	"L L 
L  &-	P	P 	P 		P 	"	P 	P 
	P$ &-
T
T 
T 	
T
 
T 	"
T 
T 

T  &-II 	"	I
 I 
I #,55 	5
 5 
5 9g 93 97 9<g <3 <C <G <@g @3 @C @S @W @Dg D3 DC DS Dc Dg D:x :S :X :=x =S =c =h =Ax AS Ac As Ax AEE#&E,/E58E>AE	E
*x * *( *8v 83 86 8;v ;3 ;C ;F ;?v ?3 ?C ?S ?V ?Cv C3 CC CS Cc Cf C*v *e * *2V 2E 2f 2 #,	?Hhx%0I'JKKL? 	?
 
? #,	?Hhx'@AAB? 	?
 
? #,	?Hhx778? 	?
 
? #,	?Hhuo-.? 	?
 
? #,	?Hhsm+,? 	?
 
? PW=&(3-/0=<DV<L=	= #*	=FHXc]334= 	=
 
= #*	=FHXhsm%<==>= 	=
 
= #*	=FHXhx}.E%FGGH= 	=
 
= PW=&(3-/0=<DV<L=	= #'	'GXg.=>' 	'
 
',9f 9Xf-= 9 9 8= 
1 
1 
1 
1 ?D:: ):7;:	: 9> !) ! !) ! 49 w D W    	
   
" AFSU%+9=MP	 79s%03GL	'28	  	
   
x			 	 		
 	 	 
	%+5=JO	x!	"% %U8V+;%< %( 6 c h h  (  :?'26	 33 3 	3
 3 3 3 
3(   	
     
(B	 Be BD BY B LQOO )O.3ODHO	O
Ni N$ N9 N <AJJ )J48J	J
R9 Rt R	 R <ANN )N48N	N
y 4 I  JO ).7BF	I    <A		 )	48			,i ,$ ,9 , <A		 )	48			i $ 9  <A )48	$Y 9 (*	 *i *

Y 
 
) 
 <A )48	 i $ 9  <A )48	 FK		',	>B			    	
  
@ JP--%,-14-AF-	-& -- - 	-
 - - - - - - 
x8X5	6-4g % G g w 5 H v ( 0h 0 0E(FBR<S 0X  56AQ;R X  8 H v %&@P:Q  /5	H V  
( 
V 
 

H 
v 
( 
(.9?		
 	
c 	
f 	

 
6 
f 
 NRLLL&)L8@8JL	L* *6 *h *66(.68@6	6
rE   r7   r  c_inith_initr   r   r>   c                    t        |       }|j                  \  }}}|j                  \  }	}
|d   }t        t        |j	                  |||j                  d   |dz  fd            }t        t        |j	                  |||j                  d   |fd            }t        t        |j	                  |||j                  d   |fd            }t        t
        |j	                  ||fd            }t        t
        |j	                  ||fd            }g }d}|D ]  }|j                  |||z   f       ||z  } d}|}t        |      D ]  }|j                  d   }
t        |      D ]  }|||f   j                  d|f      }|||f   j                  d|f      }t        | |||
|      \  }}t        |      \  }}}|||fxx   |j                  ||j                        z  cc<   |||fxx   |z  cc<   |dk(  r|n
t        |      D ]|  \  }}|d ||z
   }|d ||z
   }|||||f   }|d |j                  d    }||j                  ||j                        z  }t        t        |j                  d|df            }t        |d d d d df         }t        |d d d d df         }t        |d d d d df         } |j!                  |d d d d df         }!||z  }"|"||!z  z  }"|j#                  ||| |!f      j                  dd|f      j%                  d	      j                  d|dz  f      }t'        ||| j                  d   z         }|j!                  |"      | z  |||||f<   ||||||f<   |"|||||f<   |"}|||||f   } ! t        t
        ||   j%                  d
      j                  |	df            }#|dk(  r|j)                  |#      }#|#} #||||ffS )Nr      r   rN   r/   r;   r   r  )r   r   r/   r   )r,   r   r   r   r   r   rb   r   r&  _split_weights_transpose_weightsr   r   reversedr   rm  r  hstackr   minrc  )$r  rq  rr  r   r   r:   depthdirsr   r\  nI
batch_sizeGr   CYt2Ct2rg   r2  params_iorig_Xrf   dlayer_paramsWxWhbiasendGt3Gt3_hfhihohcCt3Hs$                                       rC   r  r    s%    
&	!BllOE4GGEArJXrxxaggaj"q& AxMNAXrxxaggaj" =SxIJAXrxxaggaj" =SxIJA
x:r"2#>
?C
x:r"2#>
?CGE 
uz123 HF5\ .WWQZt (	)A A,&&2w/CA,&&2w/C%3FAr2x%P"L(-l;LBDadGrvva&GadGtOG)*agXg5F )
s-C%K(-C%K(1eCi(.CIIaL)rvvc244((Hckk2r1+&>?T!Q']+T!Q']+T!Q']+WWT!Q']+3hrBw IIr2r2./Wb!R[)Yy)Wb"q&\*	  #urxx{23%'WWS\B%6!Qc	/"%(!Qc	/"%(!Qc	/"1eCi(?)(	)R 1Q4>>)4<<aWEF19$$Q'A].^ q!QrE   r   r  c                    t        |      }|\  }}}}|j                  \  }	}
}}|j                  d   }|d   }t        t        |j	                  ||f|j
                              }t        t        |j	                  ||dz  f|j
                              }t        t        |j	                  |j                  d   f|j
                              }d}g }t        |	      D ]b  }|j                  g        |dk(  r|n||
z  }t        |
      D ]5  }t        |||||      \  }}t        |      }|d   j                  ||f       7 d d}g }t        |	      D ]b  }|j                  g        |dk(  r|n||
z  }t        |
      D ]5  }t        |||||      \  }}t        |      }|d   j                  ||f       7 d g }d}|D ]  }|j                  |||z   f       ||z  } |gt        |	dz
        D cg c]5  }t        t        ||   j                  d      j                  |df            7 c}z   }|D cg c];  }|j	                  |j                  d   |j                  d   f|j
                        = }}t        t        |	            D ]~  }t        t        | j                  ||
|f      j                  d            }||   }||   }|
dk\  r|j                  |      }t        |
      D ]  }||   |   d   \  } }!}"||   |   d   \  }#}$}%|dk(  r|d   \  }&}'|d d }(|(j!                          n|d   \  }&}'|dd  }(|(D ]}  \  })}*t#        |*|)z
  |'|&z
        }+t%        |||&|&|+z   f   ||&|&|+z    ||||&|&|+z   f   ||||&|&|+z   f   ||||)|)|+z   f         \  },}-|||)|)|+z   fxx   |,|!z  z  cc<   |-||)|)|+z    |)}&|*}' |#|j&                  |z  z  }#|$|j&                  |||f   z  z  }$|%|j)                  d      z  }%||| z  z  } |}  j                  d   j                  d   k(  sJ g }.|D ]&  }|D ]  \  }/}0|.j                  t+        |/             ! ( ||j-                  |.      fS c c}w c c}w )	Nr/   r   rN   rt  r;   r   r   r  )r,   r   r   r   r   rO   r   r   rb   ru  rv  r   r&  rw  r   rc  r   ry  backprop_lstm_gatesr   r   _untranspose_unsplit_weightsr   )1r   r   r  r  r:   r   r~  r  r   rz  r{  r\  r   r|  r}  dCdGr  r  all_layer_paramsrf   n_inputsr  r  all_layer_gradslayer_gradsrg   r2  r  dXsr   r   r  r  r  dWxdWhd_biasstart_t3end_t3layer_indicesstart_t2end_t2rY   dGt3dCt2
grad_parts	dir_gradsr   s1                                                    rC   r  r    sF   	&	!B JAq!QE4B	
BJ	h!R8	9B	h!R!VAGG<	=BHbhhQ'9hNOH HTV5\ B#a2b4it 	BA%3FAr8X%V"L(-l;LR ''x(@A	BB HSUO5\ @r"a2b4it 	@A$28QHh$W!K,[9KB&&X'>?	@@ GE 
uz123 uqy! 	Xqt~~i088!RAB 
B GI
I288QWWQZ,AGG8<
IC
IeEl# 'Hbjj!T27AA)LMVqE19''-Dt  	A+A.q1!4LBD.q1!4Q7CfAv#*2; & '%%'#*1: & '$1   &6H,fx.?@0Hx$667x(T/2aHx$667aHx$667aHx$667
d Q8d?223tby@3158ho.#   244!8OC244!AqD'>!Cbff!fn$F"r'MBA 	B O'P 88A;!''!*$$$J& G' 	GLIq:9EF	GG r~~j)))g Js   .:Q*1A Q/rf   r   r|  r  c                 N   d|z  |z  }d|z  }d|z  |z  }d|z  }| |||z    j                  d|z  |f      }	||z  }| |||z    j                  d|z  f      }
||z  }| |||z    j                  d|z  |f      }||z  }| |||z    j                  d|z  f      }||z  }|	|
f||ff|fS )Nrt  r8  )r  rf   r   r|  r  Wx_sizebx_sizeWh_sizebh_sizer  bxr  bhs                rC   ru  ru  <  s    "frkG"fG"frkG"fG	8g-	.	6	6B|	DBH	8g-	.	6	6By	ABH	8g-	.	6	6B|	DBH	8g-	.	6	6By	ABHHr2h))rE   c                    | \  \  }}\  }}t        |      }|j                  dd|j                  d   f      }|j                  d      j                  d|j                  d   f      }|j                  d      j                  d      j                  d      }|j                  dd|j                  d   f      }|j                  d      j                  d|j                  d   f      }|j                  d      j                  d      j                  d      }|j                  } ||      } ||      } ||      |z   }|||fS )Nrt  r;   r   )rt  r;   r/   r   r;   )r,   r&  r   r   rc  )r  r  r  r  r  r:   ascontigr  s           rC   rv  rv  L  s%     HRhr2	"	B	QBHHRL)	*B	i	 	(	("bhhrl);	<B	G		&	&v	.	6	6u	=B	QBHHRL)	*B	i	 	(	("bhhrl);	<B	G		&	&v	.	6	6u	=B##H	"B	"BB<"Dr4<rE   c                 *   | \  }}}t        |      }|j                  d   }|j                  d   }|j                  dd|f      j                  d      j                  d|f      }|j                  dd|f      j                  d      j                  d|f      }|j                  d      j                  d      j                  d      }|j	                  |j                  d	      }|j                  |j                         ||j                         |f      S )
Nr/   r;   rt  r   )r;   rt  r  r  r   rN   )r,   r   r&  r   r   r   ravel)r  r  r  r  r:   r   r|  r   s           rC   r  r  ^  s    LBD	"	B	!B	!B	RBK	 	*	*9	5	=	=r2h	GB	RBK	 	*	*9	5	=	=r2h	GB<< **62::5ADHHTZZsH+E>>288:tRXXZ?@@rE   dYt3dCt3r  r  r  c                    t        |       }|j                  |dd      \  }}}}	|j                  d   |j                  d   cxk(  r&|j                  d   cxk(  r|	j                  d   k(  sJ  J |j                  d   | j                  d   cxk(  r9|j                  d   cxk(  r&|j                  d   cxk(  r|j                  d   k(  sJ  J |j                  |      }
| |
z  }| |z  }||t	        |
      z  z  }||	z  }||z  }||z  }||z  }|t	        |	      z  }|t        |      z  }|t        |      z  }|t        |      z  }|j                  ||||fd      }||fS )Nrt  r;   r  r   )r,   r   r   r  ru  ro  r   )r  r  r  r  r  r:   r  r  r  r  tanhCt3d_ho	d_tanhCt3d_hid_hcd_hfr  d_At3_hcd_At3_hod_At3_hid_At3_hfdAt3s                         rC   r  r  j  sg    
$	BXXc12X.NBB88A;"((1+C!CCCCCC88A;$**Q-X4::a=XCIIaLXCIIVWLXXXXXggclG'>Dr	IIg&&D"9D"9D#:D"9DeBiHhrl"Hhrl"Hhrl"H>>8XxB>LD:rE   c                 n    t        |       }|j                  | dd      } dd|j                  |        z   z  S )Nrg  rh  r5   )r,   rj  rk  )r   r   r:   s      rC   rm  rm    s:    	!	B 	5$A#r
"##rE   r   c                     | d| z
  z  S r   r   r   s    rC   ro  ro    s    a=rE   c                     d| dz  z
  S )Nr/   r   r   r  s    rC   ru  ru    s    q!t8OrE   opsc                 >    dd| j                  t        |z        z   z  S )z6Gaussian CDF for distribution with mean 0 and stdev 1.r  r5   )r  	INV_SQRT2r  r   s     rC   r  r    s    #	A..//rE   c                 R    t         | j                  j                  d|z  |z        z  S )z6Gaussian PDF for distribution with mean 0 and stdev 1.g      )INV_SQRT_2PIr:   rk  r  s     rC   r  r    s"    #&&**TAX\222rE   uvc                     | j                   |j                   k7  r'd| j                    d|j                    }t        |      y )Nr  r  )r   rU   )r  r  r  s      rC   r  r    s;    ww!''1!''%yIo rE   r@   )Hrl   mathtypingr   r   r   r   r   r   r	   r
   r   r   r   rQ   typesr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   utilr,   r-   r.   rK   r0   r1   r3   r  pir  r  r  r7   r  r  ra   ru  rv  r  r  rm  ro  ru  r  r  r  r   rE   rC   <module>r     s               > ; : 		)
)7
+
$))C$''M
")$))C. 	YTYYsTWW}--j jZ)#LD D &D 08D =ED PVD 
8U?D Nc*h c* c* c*e c*L*8 * * *# * * $	A
")18@GO
8X8$ 6 V  0c 0i 0I 0
3c 3i 3I 3
x H rE   