
    is                     '   d dl Z d dlZd dlmZmZ d dlZd dlZd dlmZm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZ d dlmZmZmZ d d	lmZm Z m!Z! d d
l"m#Z# d dl$m%Z%m&Z& ddl'm(Z( ddl(m)Z)m*Z* dZ+ ee      Z, e       Z- ed      Z.e-e,gZ/e-gZ0ere0jc                   e              e0e,gz   Z2ddgZ3ddgZ4g dZ5g dZ6d Z7e r e7       Z8ng Z8ejr                  ju                  deeg      d        Z;ejr                  ju                  de2      ejr                  jy                  d      d               Z=ejr                  ju                  de2      d        Z>ejr                  ju                  de0      d        Z?ejr                  ju                  de0      d         Z@ejr                  ju                  de0      d!        ZAejr                  ju                  de2      ejr                  ju                  d"e3      ejr                  ju                  d#dd$g      d%                      ZBejr                  ju                  de0       e e(jR                         &      d'               ZCejr                  ju                  de2      d(        ZDejr                  ju                  de/      d)        ZEejr                  ju                  de2      ejr                  ju                  d"e3       e	e+d*       e e(j                         +      d,                             ZGejr                  ju                  de2      ejr                  ju                  d"e3      d-               ZHejr                  ju                  de2       e	e+d*       e e(jR                         +      d.                      ZIejr                  ju                  de0      ejr                  ju                  d"e3      d/               ZJejr                  ju                  de0      ejr                  ju                  d"e3      d0               ZKejr                  ju                  de0      ejr                  ju                  d"e3      d1               ZLejr                  ju                  de0      ejr                  ju                  d"e3      d2               ZMejr                  ju                  de0      ejr                  ju                  d"e3      d3               ZNejr                  ju                  de0      ejr                  ju                  d"e3      d4               ZOejr                  ju                  de2      ejr                  ju                  d"e3       e	e+d*       e e(jR                         +      d5                             ZPejr                  ju                  de0      ejr                  ju                  d"e3      d6               ZQejr                  ju                  de0      ejr                  ju                  d"e3      d7               ZRejr                  ju                  de0      ejr                  ju                  d"e3      d8               ZSejr                  ju                  de0      ejr                  ju                  d"e3      d9               ZTejr                  j                  e d:;      ejr                  ju                  d<d=dg      d>               ZVejr                  j                  e d:;      ejr                  ju                  d<d=dg      d?               ZWejr                  ju                  de2      ejr                  ju                  d"e3       e	e+d*       e e(jR                         +      d@                             ZXejr                  ju                  de2       e	e+d*       e e(jR                         +      dA                      ZYejr                  ju                  de2       e	e+d*       e e(jR                         +      dB                      ZZdCe#dDe#dEe[dFee#e#f   fdGZ\ejr                  j                  e  dH;      ejr                  ju                  de2      ejr                  ju                  dEg dI      dJ                      Z]ejr                  ju                  dKg e/e.      dL        Z^ejr                  ju                  dKg e/e.      dM        Z_ejr                  ju                  dKe/       e	e+d*       e e(jR                         +      dN                      Z`ejr                  ju                  de2      ejr                  ju                  d"e3e4z         dO               Zaejr                  ju                  de2      ejr                  ju                  d"e3      dP               Zbejr                  ju                  de2      ejr                  ju                  d"e3      dQ               Zcejr                  ju                  de2      ejr                  ju                  d"e3      dR               Zdejr                  ju                  de2      ejr                  ju                  d"e3      dS               Zeejr                  ju                  de2      ejr                  ju                  d"e3      dT               Zfejr                  ju                  de2      ejr                  ju                  d"e3      dU               Zgejr                  ju                  de2      ejr                  ju                  d"e3      dV               Zhejr                  ju                  de2      ejr                  ju                  d"e3      dW               Ziejr                  ju                  de2      ejr                  ju                  d"e3      dX               Zjejr                  ju                  de2      ejr                  ju                  d"e3      dY               Zkejr                  ju                  de2      ejr                  ju                  d"e3      dZ               Zlejr                  ju                  de2      ejr                  ju                  d"e3      ejr                  ju                  d[e5      d\                      Zmejr                  ju                  de2      ejr                  ju                  d"e3      ejr                  ju                  d[e5      d]                      Znejr                  ju                  de2      ejr                  ju                  d"e3      ejr                  ju                  d^e6      d_                      Zoejr                  ju                  de2       e	e+d*       e e(jR                         +      d`                      Zpejr                  ju                  de0      ejr                  ju                  d"e3      ejr                  ju                  dg da      db                      Zqejr                  ju                  de2       e	e+d*       e e(jR                         +      dc                      Zrejr                  ju                  de2       e	e+d*       e e(jR                         +      dd                      Zsejr                  ju                  de2       e	e+d*       e e(jR                         +      de                      Ztejr                  ju                  de2       e	e+d*       e e(jR                         +      df                      Zuejr                  ju                  de2       e	e+d*       e e(jR                         +      dg                      Zvejr                  ju                  de2       e	e+d*       e e(jR                         +      dh                      Zwejr                  ju                  de2       e	e+d*       e e(jR                         +      di                      Zxejr                  ju                  de2       e	e+d*       e e(jR                         +      dj                      Zyejr                  ju                  de2       e	e+d*       e e(jR                         +      dk                      Zzejr                  ju                  de2       e	e+d*       e e(jR                         +      dl                      Z{ddmZ|edn        Z}ejr                  ju                  de0      ejr                  ju                  dog dp      dq               Z~ejr                  j                  dr;      ejr                  ju                  de0       e	e+d*       e e}       s      dt                             Zdu Zdv Zdw Zdx Zdy Zejr                  j                  e  dH;      ejr                  j                  e! edz      k  d{;      ejr                  ju                  de2      ejr                  ju                  d"ddg      ejr                  ju                  d|e8       e	e+d*       e e(j                  d}d~       e(j                  dd=            d                                                  Zejr                  ju                  de2       e	e+d*       e e(j                  dd            d                      Zejr                  ju                  de2      ejr                  ju                  dd       e	e+d*       e e(j                  dd            d                             Zejr                  j                  e d:;      d        Zejr                  ju                  de2      d        Zy)    N)Tuplecast)givensettings)	compositeintegers)assert_allclose)Version)LSTMCupyOpsNumpyOpsOpsfix_random_seedget_current_opsget_opsuse_ops)KERNELSKERNELS_LISTcompile_mmh)has_cupy_gpu	has_torchtorch_version)Floats2d)torch2xpxp2torch   )
strategies)	arrays_BIndarrays_of_shape
   T)use_blisfloat32float64int32int64)reduce_firstreduce_last
reduce_maxreduce_mean
reduce_sum))r&   T)r'   T)r(   T)r)   F)r*   Fc                      dd l dd lfd} fd}fdfd}fd}fd}fd}fd	}fd
}d }fd}	fd}
d| fd|fdfd|fd|fd|fd|fd|fd|fd|	fd|
fd|fgS )Nr   c                 N    j                   j                  j                  |       S N)nn
functionalreluxtorchs    n/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/thinc/tests/backends/test_ops.py
torch_reluz(create_pytorch_funcs.<locals>.torch_relu@       xx""''**    c                 N    j                   j                  j                  |       S r-   )r.   r/   relu6r1   s    r4   torch_relu_kz*create_pytorch_funcs.<locals>.torch_relu_kC   s    xx""((++r7   c                 6    j                  | dz  dz   dd      S )N皙?      ?r      )clipr1   s    r4   torch_hard_sigmoidz0create_pytorch_funcs.<locals>.torch_hard_sigmoidF   s    zz!c'C-A..r7   c                 N    j                   j                  j                  |       S r-   )r.   r/   hardtanhr1   s    r4   torch_hard_tanhz-create_pytorch_funcs.<locals>.torch_hard_tanhI   s    xx""++A..r7   c                 N    j                   j                  j                  |       S r-   )r.   r/   mishr1   s    r4   
torch_mishz(create_pytorch_funcs.<locals>.torch_mishL   r6   r7   c                 N    j                   j                  j                  |       S r-   )r.   r/   silur1   s    r4   torch_swishz)create_pytorch_funcs.<locals>.torch_swishO   r6   r7   c                     |  |       z  S r-    )r2   r@   s    r4   torch_hard_swishz.create_pytorch_funcs.<locals>.torch_hard_swishR   s    %a(((r7   c                 N    j                   j                  j                  |       S r-   )r.   r/   	hardswishr1   s    r4   torch_hard_swish_mobilenetz8create_pytorch_funcs.<locals>.torch_hard_swish_mobilenetU   s    xx"",,Q//r7   c                 &    j                  |       S r-   )sigmoidr1   s    r4   torch_sigmoidz+create_pytorch_funcs.<locals>.torch_sigmoidX   s    }}Qr7   c                 F    d| z  | d| | z  z   j                         z  dz   z  S )Nr=   r>   )sqrtr2   s    r4   
torch_dishz(create_pytorch_funcs.<locals>.torch_dish[   s,    Qw!q1q5y..001455r7   c                     d| z  dj                  j                  dj                  z        | dj                  | d      z  z   z        z   z  S )Nr=         ?       @gHm?      @)tanhrT   pipow)r2   mathr3   s    r4   torch_gelu_approxz/create_pytorch_funcs.<locals>.torch_gelu_approx_   s\     **IIcDGGm,HuyyC?P4P0PQ		
r7   c                 N    j                   j                  j                  |       S r-   )r.   r/   gelur1   s    r4   
torch_geluz(create_pytorch_funcs.<locals>.torch_geluk   r6   r7   r0   relu_khard_sigmoid	hard_tanhrE   swish
hard_swishhard_swish_mobilenetdishgelu_approxra   rQ   )r^   r3   )r5   r:   rC   rF   rI   rL   rO   rR   rV   r_   rb   r^   r3   r@   s              @@@r4   create_pytorch_funcsrk   ;   s    +,//++)0 6

+ 
	< 	+,	o&		+	'(	!;<		)*		M" r7   opc                    t        |       D cg c]  }|j                  d      r| }}|D ]  }t        t        |      sJ t	        | |      }t        |d      s/t        j                  |      }|j                  D cg c]  }| c}dd }t        j                  t	        t        |            }t        |       |j                  D cg c]  }| c}dd }	||	k(  sJ |       |j                  j                         D cg c]  }|j                   c}dd }
|j                  j                         D cg c]  }|j                   c}dd }|
|k(  sJ |       |j                  j                         D cg c]  }|j                   c}dd }|j                  j                         D cg c]  }|j                   c}dd }t        t        ||            D ]D  \  }\  }}|t
        j                  j                  k7  s'|t
        j                  j                  k7  sEF  yc c}w c c}w c c}w c c}w c c}w c c}w c c}w )zTest that specific ops don't define any methods that are not on the
    Ops base class and that all ops methods define the exact same arguments.___call__r>   N)dir
startswithhasattrr   getattrinspect	signature
parametersprintvaluesdefault
annotation	enumeratezip	Parameterempty)rl   mattrsattrmethodsigpparamsbase_sigbase_paramsdefaultsbase_defaultsannotsbase_annotsip1p2s                    r4   test_ops_consistencyr      s    B91q||C'8Q9E9 sD!!!T"6:&##F+C!$0Aa04F((d);<H(O&.&9&9:1:12>K[(.$.(+.>>+@+@+BCa		CABGH080C0C0J0J0LM1QYYMabQM},2d2,,/NN,A,A,CDqallDQRHF191D1D1K1K1MNA1<<NqrRK(V[)AB 8B**000R7;L;L;R;R5R
 ! : 1 ;CM ENs.   H%H%?	H*
	H/H46H97H>,Iopszignore::RuntimeWarningc                    | j                   j                  dd      }| j                   j                  dd      }| j                  ||||dddd       t        j                  t
              5  | j                  ||||dddd       d d d        t        j                  t
              5  | j                  ||||dddd       d d d        t        j                  t
              5  | j                  ||||dddd       d d d        t        j                  t
              5  | j                  ||||dddd       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   \xY w# 1 sw Y   y xY w)Nr>   fdtyper           )xpzerosadampytestraises
ValueError)r   onetwos      r4   test_adam_incorrect_inputsr      s;    &&,,q,
$C
&&,,q,
$CHHS#sCc34	z	" 9c3S#sC89	z	" 9c3S#sC89	z	" 9c3S#sC89	z	" 9c3S#sC89 99 99 99 99 9s0   ,D<&E EE <EEE E)c                    | j                   | j                  | j                  | j                  f}t	        |      D ]  \  }}d|dz   z  } || }|j
                  t        j                  k(  sJ |j                  t        |      k(  sJ | j                  |      }|j
                  t        j                  k(  sJ |j                  t        |      k(  rJ  | j                  | j                  | j                  | j                  f}t	        |      D ]  \  }}d|dz   z  } || }|j
                  t        j                  k(  sJ |j                  t        |      k(  sJ | j!                  |      }|j
                  t        j                  k(  sJ |j                  t        |      k(  rJ  | j#                  d      j                  dk(  sJ y )N)r>   r>   )alloc1falloc2falloc3falloc4fr{   r   numpyr"   ndimlenalloc_falloc1ialloc2ialloc3ialloc4ir$   alloc_ialloc)r   float_methodsr   r   shapearrint_methodss          r4   
test_allocr      s   [[#++s{{CKKHM}- &	6AenyyEMM)))xx3u:%%%kk% yyEMM)))xx3u:%%%& ;;S[[#++FK{+ &	6AenyyEKK'''xx3u:%%%kk% yyEKK'''xx3u:%%%& 99Q<!!!r7   c                    | j                  dd      }| j                  |d      }|j                  dk(  sJ |j                  dk(  sJ t	        t        |            D ]+  }t	        |j                  d         D ]  }|||f   dk7  rJ  - y )N   uint64r   r   )r      uint32r>   )r   hashr   r   ranger   )r   idskeysr   js        r4   test_hash_gives_distinct_keysr      s    
++ax+
(C88CD::::!!!3s8_ #tzz!}% 	#A1:?"?	##r7   c                     d}d}| j                  ||      }|dk  r||dk(     j                         sJ y ||dk7     j                         sJ y )Nr   r   r   rX   )get_dropout_maskallr   r   dropmasks       r4   test_get_dropout_emptyr      s[    EDt,Ds{DCK $$&&&DCK $$&&&r7   c                     d}d}| j                  ||      }|dkD  j                         sJ |dk(  j                         sJ |j                  |k(  sJ y )N)   r   r=   rX   r   )r   anyr   r   s       r4   test_get_dropout_not_emptyr      s\    EDt,D3JCK::r7   r   index_dtyper   c                 ,   | j                   j                  d|      j                  dd      }| j                   j                  ddgddgddgg|      }| j	                  ||      }| j                   j
                  j                  |g dg d	g d
g       y )N   r   r      r   r   r>   )      @       @      $@)      (@g      ,@      0@)rZ         @      @)r   arangereshapearray
gather_addtestingr	   )r   r   r   tableindicesgathereds         r4   test_gather_addr      s     FFMM"EM*221a8EffllQFQFQF3;lGG~~eW-HFFNN""#%7Ir7   )r   c                 4   | j                  |      }| j                  j                  dd      j                  dd      |j                  d   z  }| j                  j
                  j                  | j                  ||      ||   j                  d      d	       y )
Nd   r   r      r   r   r>   h㈵>atol)	asarrayr   r   r   r   r   r	   r   sumr   r   r   s      r4   test_gather_add_against_numpyr      s     KKEffmmCsm+33B:U[[^KGFFNN""ug&g1 # r7   c                 *   | j                   j                  dd      j                  dd      }| j                   j                  ddgddgd	dggd
      }t	        j
                  t              5  | j                  ||       d d d        y # 1 sw Y   y xY w)Nr   r   r   r   r   r   r   r>   r   r   )r   r   r   r   r   r   
IndexErrorr   r   s      r4   test_gather_add_oob_raisesr     s}    FFMM"CM(00A6EffllQFQFQF33l?G	z	" 'ug&' ' 's   -B		Bc                    | j                  dgdgdgdggd      }| j                  |d      }t        |d      r|j                         }t	        |d	   g d
       t	        |d   g d       t	        |d   g d       t	        |d   g d       y )NrX   rZ         @r   r"   r   r>   getr   )r   rX   rZ   )rX   rZ   r   r   )rZ   r   r   r   )r   r   r   )r   seq2colrr   r   r	   )r   seqcolss      r4   test_seq2col_window_one_smallr   
  s    
++usecUQC0	+
BC;;sADtUxxzDG_-DG_-DG_-DG_-r7   )max_examplesdeadlineXc           	         | j                  ||      }|j                  d      j                  |      }| j                  |      \  }}|j                  |k(  sJ | j
                  j                  j                  ||dd       | j
                  j                  j                  | j
                  j                  || j
                  j                  |d      d      | j
                  j                  |d      d       y )Nr   axisMbP?rtolr   g|=r   )
r   maxastypemaxoutr   r   r   r	   take_along_axisexpand_dims)r   r   r   expected_bestpredicted_bestwhichs         r4   test_maxoutr    s    
 	AU#AEErEN))%0MJJqMNE5(((FFNN""~E #  FFNN""q#&&"4"4UB"?bI="- # r7   c           	         | j                  | j                  ddgddgg|      | j                  ddgddgg      d	      }|j                  |k(  sJ | j                  j
                  j                  |g d
g dgg dg dgg       t        j                  t              5  | j                  | j                  ddgddgg      | j                  ddgd	dgg      d	       d d d        y # 1 sw Y   y xY w)NrX   rY   rZ   r   r   r>   r   r   r   )r   rX   r   rY   r   r   r   r   rZ   )r   r   r   )
backprop_maxout	asarray2f	asarray2ir   r   r   r	   r   r   r   r   r   dXs      r4   test_backprop_maxoutr  -  s     
		SzC:.e<1v1v&'	
B
 88uFFNN""

?	+o-OP
 
z	" 
MMC:Sz23S]]QFQPQFCS5TVW	

 
 
s   >C%%C.c                 T   | j                  |      }t               }| j                  |_        |j                  |j                        |z   }|j                  |j                  |      d      }| j                  |d      }| j                  j                  j                  ||dd       y )Nr>   nWr   r   r   )r   r   r   r   r   r   r   r	   )r   r   base_opsbaseXtarget	predicteds         r4   test_seq2col_window_oner  A  s     	AAuH&&HKNN177#a'Eh..u5!<FA!$IFFNN""695u"Mr7   c                 ,   | j                   j                  j                  | j                  d|      | j	                  | j                  d|      d| j                   j                  dd                   | j                   j                  j                  | j                  d|      | j                  | j                  d|      d| j                   j                  dd                   | j                   j                  j                  | j                  d|      | j	                  | j                  d|      d| j                  dg                   | j                   j                  j                  | j                  d|      | j                  | j                  d|      d| j                  dg                   | j                   j                  j                  | j                  d|      | j	                  | j                  d|      d| j                  ddg                   | j                   j                  j                  | j                  d|      | j                  | j                  d|      d| j                  ddg                   y )N)r   r   r   r>   )r   r$   lengthsr   )r   r   r	   r   r   r   backprop_seq2col	asarray1ir   r   s     r4   test_seq2col_lengths_all_zeror  N  s    FFNN""		&	&IIfEI*Asvv||DPW|7X 	 	
 FFNN""		&	&IIfEI*Asvv||DPW|7X 	 	
 FFNN""		&	&CIIfEI2As}}aS?QR
 FFNN""		&	&IIfEI*As}}aS7I 	 	
 FFNN""		&	&CIIfEI2As}}aQRV?TU
 FFNN""		&	&IIfEI*As}}aV7L 	 	
r7   c                    | j                  g dg dg dg dg dg|      }| j                  g dg dg d	g d
g dg|      }| j                  j                  j                  || j	                  | j                  j                  dd|      j                  dd      d| j                  ddg                   | j                  j                  j                  || j                  |d| j                  ddg                   | j                  j                  j                  || j	                  | j                  j                  dd|      j                  dd      d| j                  ddg                   y )N	r   r   r   r>   r   r   r   r      	r>   r   r   r   r   r!        	   	r   r   r!  r#  r$  r%  r       r   	r#  r$  r%  r    r'  r            	r    r'  r   r)  r*  r+  r   r   r   r   r   r   r!  r   r+                 !   $         r5  rX   r   r   r   r>   r   r  	r
  r   r   r	   r   r   r   r  r  r   r   
cols_check
grad_checks       r4   $test_seq2col_lengths_zero_first_lastr?  {  s^    ''*--	
   	J 	L,lKSX  J
 FFNN""FFMM#t5M199!Q?MM1a&) 	 	
 FFNN""MM1a&) 	 	
 FFNN""FFMM#t5M199!Q?MM1a&) 	 	
r7   c                    | j                  g dg dg dg dg dg dg dg|      }| j                  g d	g d
g dg dg dg dg dg|      }| j                  j                  j                  || j	                  | j                  j                  dd|      j                  dd      d| j                  g d                   | j                  j                  j                  || j                  |d| j                  g d                   | j                  j                  j                  || j	                  | j                  j                  dd|      j                  dd      d| j                  g d                   | j                  j                  j                  || j                  |d| j                  g d                   y )Nr   r"  r&  r(  r,  )	r   r   r         r/        r1  )	rA  rB  r/  rC  rD  r1  r   r   r   r   r-  r.  r0  r4  r8  )    "   r7  )&   (   *   rX   g      6@r#  r   r>   )r   r   r   r  )r   r   r   r   r;  r<  s       r4   !test_seq2col_lengths_zero_betweenrJ    s    ''*----	
   J 	
   J FFNN""FFMM#t5M199!Q?MM), 	 	
 FFNN""MM), 	 	
 FFNN""FFMM#t5M199!Q?MM,/ 	 	
 FFNN""MM,/ 	 	
r7   c           
      :   | j                   j                  dd|      j                  dd      }| j                  g d      }| j	                  |d|      }| j                   j
                  j                  | j                  g d	g d
g dg dg dg|      |       y )NrX   r   r   r   r   r>   r   r>   r>   r  )	r   r   r   r>   r   r   r   r   r   )	r   r   r   r   r   r!  r#  r$  r%  r&  )	r#  r$  r%  r    r'  r   r   r   r   )	r   r   r   r)  r*  r+  r   r   r   r   r   r   r  r   r   r	   r
  r   r   r   r  r   s        r4   test_seq2col_window_one_lengthsrO    s     	c4u-55a;AmmI&G;;q!W;-DFFNN""++...  	 		
 	r7   c           
      :   | j                   j                  dd|      j                  dd      }| j                  g d      }| j	                  |d|      }| j                   j
                  j                  | j                  g d	g d
g dg dg dg|      |       y )NrX   r   r   r   r   rL  r   r  )r   r   r   r   r   r   r>   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r!  r#  r$  r%  r    r'  r   )r   r   r   r   r   r!  r#  r$  r%  r    r'  r   r   r   r   )r   r   r!  r#  r$  r%  r    r'  r   r   r   r   r   r   r   )r   r   r   r   r   r   r)  r*  r+  r   r   r   r   r   r   rM  rN  s        r4   test_seq2col_window_two_lengthsrQ    s     	c4u-55a;AmmI&G;;q!W;-DFFNN""=@@@@  	 		
 	r7   c                     | j                  g dg dg dg|      }dgdgdgg}| j                  |d      }t        |t        j                        s|j                         }t        ||d	d	
       y )N)r   r   r   )      r   rX   r  r   rS  rY   rX   r>   r   r  )r   r  
isinstancer   ndarrayr   r	   r   r   r   expectedr   s        r4   &test_backprop_seq2col_window_one_smallrX    sq     ;;	*O<E  D u%H


tQ
'Cc5==)ggiCE:r7   c                 l   |j                   d   dz  ry | j                  ||      }| j                  j                  |      j	                         dk\  ry t               }| j                  |_        |j                  |d      }| j                  |d      }t        |j                   d         D ]\  }||   j                         ||   j                         z
  }|dk  s|dkD  s5t        ||       t        ||          t        ||          ^ | j                  j                  j                  ||d	d	
       y )Nr>   r   r   r5  r  r   g皙?r   r  )r   r   r   absr   r   r  r   r   rw   r   r	   )r   r   r   r  r  r  rowdiffs           r4    test_backprop_seq2col_window_oner^  &  s   
 	wwqzA~AU#A
vvzz!}b uH&&HK&&qQ&/F$$Q1$-IV\\!_% "c{ 9S>#5#5#77$;$*#t&+)C.!" FFNN""695u"Mr7   c           
      @   | j                   j                  ddd|      j                  dd      }| j                  g d      }| j	                  |d|      }| j                   j
                  j                  | j                  g d	g d
g dg dg dg|      |d       y )NrZ  gffffff@stepr   r   r%  rL  r>   r  )g?r=   g333333?)g	@g333333@g@)gffffff@皙@g@)gffffff@g333333@r   )r   gffffff@g@r   ư>r   r   r   r   r  r  r   r	   r
  r   r   d_yr  d_seqss        r4   (test_backprop_seq2col_window_one_lengthsrh  =  s     &&--Ss%-
8
@
@A
FCmmI&G!!#q'!:FFFNN""  	 		
 	 # r7   c                 :   | j                  dgdgdgdgg|      }| j                  |d      }t        |t        j                        s|j                         }t        |d   g d       t        |d	   g d
       t        |d   g d       t        |d   g d       y )NrX   rY   rZ   r   r   r   r   r   r   rX   rY   rZ   r>   r   rX   rY   rZ   r   rX   rY   rZ   r   r   r   rY   rZ   r   r   r   )r   r   rT  r   rU  r   r	   )r   r   r   r   s       r4   test_seq2col_window_tworn  T  s     ++usecUQC0+
>C;;sADdEMM*xxzDG67DG67DG67DG67r7   c           
      <   | j                   j                  ddd|      j                  dd      }| j                  g d      }| j	                  |d|      }| j                   j
                  j                  | j                  g d	g d
g dg dg dg|      |       y )NrZ  gffffff@r`  r   r+  rL  r   r  )gffffff?g?g?)gffffff$@g      %@g%@)g333333&@g&@gffffff'@)r   g(@g333333)@)g@g333333@rb  r   rd  re  s        r4   (test_backprop_seq2col_window_two_lengthsrp  a  s     &&--Ss%-
8
@
@B
GCmmI&G!!#q'!:FFFNN"""""  	 		
 	r7   c                     | j                  g dg dg dg dg|      }| j                  dgdgdgdgg|      }| j                  |d	      }| j                  j                  j	                  ||d
d
       y )Nrj  rk  rl  rm  r   rZ   r   r   r   r   r  )r   r  r   r   r	   rV  s        r4    test_backprop_seq2col_window_tworr  w  s     ;;%%%%		
   D {{"#"#"#		
   H 

tQ
'CFFNN""3u5"Ir7   zneeds GPU/CuPy)reasonr  r>   c                    t               }t               }d}|j                  j                  j	                  |dz        j                  d      j                  dd      }|j                  |      }|j                  g d|dz  z        }|j                  |      }|j                  || |      }|j                  || |      }	t        ||	j                                y )N   r   r"   r   r>   r   r   r>   r$  r  r  )r   r   r   randomrandnr   r   r
  r  r   r	   r   )
r  cupy_ops	numpy_ops
batch_sizer   X_gpur  lengths_gpur   cols_gpus
             r4   "test_large_seq2col_gpu_against_cpur    s     yH
I J!!*q.188CKKBPQRAq!E !!,*/"BCG$$W-KQ2w7D"kBHD(,,.)r7   c                    t               }t               }d}d| z  dz   }|j                  j                  j	                  ||z        j                  d      j                  d|      }|j                  |      }|j                  g d|dz  z        }|j                  |      }|j                  || |      }	|j                  || |      }
t        |	|
j                                y )	Nru  r   r>   r"   r   rv  r$  rw  )r   r   r   rx  ry  r   r   r
  r  r  r	   r   )r  rz  r{  r|  nFd_cols
d_cols_gpur  r~  rg  
d_seqs_gpus              r4   +test_large_backprop_seq2col_gpu_against_cpur    s     yH
I J	
R!B!!*r/299)DLLRQST  ##F+J !!,*/"BCG$$W-K''2w'GF**:"k*RJFJNN,-r7   c                    | j                  ||      }| j                  j                  |      j                         dk\  ry | j                  dgt	        |      z  d      }| j                  ||      }|j                  |k(  sJ |j                  t        |      |j                  d   fk(  sJ d}t        |      D ]O  \  }}| j                  j                  j                  ||||z    j                  d      ||   |z  dd	       ||z  }Q y )
Nr   r   r   r   r>   r   r   g{Gz?r   )r   r   r[  r   r   backprop_reduce_sumr   r   r   r{   r   r	   )r   r   r   r  outstartr   lengths           r4   test_backprop_reduce_sumr    s   
 	AU#A
vvzz!}akk1#A,ck2G

!
!!W
-C9999Wqwwqz2222Ew' 	6&&'+++3QqTF]TX 	' 	
 		r7   c                     | j                  | j                  |            }|D ]   }d|j                         cxk  rdk  rJ  J  y )NwJ?gqh ?)softmaxr   r   )r   r   yr\  s       r4   test_softmax_sums_to_oner    sH     	CKKN#A .#'')-v-----.r7   c                     | j                  |      }| j                  |d      }|D ]   }d|j                         cxk  rdk  rJ  J  y )NTinplacer  grZ|
 ?)r   r  r   )r   r   r\  s      r4   test_softmax_works_inplacer    sQ     	AAAt$A /#'').w...../r7   r   dYtemperaturereturnc           
      T   dd l }t        | d      }t        |      }||z  }|j                  j                  j	                  |d      }|j                  |       t        t        t        |            t        t        t        t        |j                  |j                                    fS )Nr   Trequires_gradr   )dim)r3   r   r.   r/   r  backwardr   r   r   Tensorgrad)r   r  r  r3   XtdYtXt_tempYts           r4   torch_softmax_with_temperaturer    s     	!4	(B
2,C;G				$	$W"	$	5BKK(2,'(4bgg67*  r7   zneeds PyTorch)r=   rX   rY   c                    | j                   j                  dddd      j                  dd      }| j                   j                  dd      }| j	                  ||      }| j                  |||      }t        |||      \  }}| j                   j                  j                  ||d       | j                   j                  j                  ||d       y )	Nr    r<   r   r   )r  rc  r   )	r   r   r   eyer  backprop_softmaxr  r   r	   )r   r  r   r  Yr  r  dXts           r4   test_softmax_temperaturer    s     	c2s#.66r2>A	Bc	"BA;/A			a		=B,QK@GBFFNN""1bt"4FFNN""2s"6r7   cpu_opsc                 "   t        j                  dd      }t        j                  dd      }|t         j                  j                  |j                        j                  |j                        z  }|t         j                  j                  |j                        j                  |j                        z  }| j                  ||d      }t        j                  ||j                        }t        ||dd	       t        j                  d
d      }t        j                  dd      }|t         j                  j                  |j                        j                  |j                        z  }|t         j                  j                  |j                        j                  |j                        z  }| j                  ||d      }t        j                  |j                  |      }t        ||dd	       | j                  ||d|       y )N)r   r   r   r   )r   r   )sizeT)trans2-C6?r  )r   r   )r   r   )trans1)r  r  )r   r   rx  uniformr  r   r   gemmdotTr	   )r  Wr   r  rW  s        r4   test_gemm_computes_correctlyr    s~   F#&AF#&A		166		*	2	2177	;;A		166		*	2	2177	;;AQ$'AyyACC HHad6F#&AF#&A		166		*	2	2177	;;A		166		*	2	2177	;;AQ$'Ayya HHad6LLAdL*r7   c                     t        j                  dd      x}}t        j                  dd      }| j                  |||       t        j                  |t        j                  d            sJ y )Nr   r   r   )r  )r   r   onesr  array_equal)r  abcs       r4   test_gemm_out_usedr    sX    KKc**A

6%ALLA1LQF 3444r7   c           	         | j                  |D cg c]  }| c}      }|j                  dk(  sJ | j                  ||D cg c]  }t        |       c}      }t	        ||       | j                  |D cg c]  }| c}dd      }t        |      t        |      kD  sJ | j                  ||D cg c]  }t        |       c}d      }t	        ||       y c c}w c c}w c c}w c c}w )Nr>   r   )padr   )r  )flattenr   	unflattenr   r	   )r  r   r2   flatunflatflat2unflat2s          r4    test_flatten_unflatten_roundtripr  '  s     ??q>!A>*D99>>ta%8c!f%89FAvOON1QNO=Eu:D	!!!':1A':BGAw *%8*':s   	CC
5	C#1C(
c           
         | j                   j                  dd|      | j                   j                  dd|      g}| j                   j                  j                  | j	                  |      g dg dg       | j                   j                  j                  | j	                  |d      g d	g d
g       | j                   j                  dd|      j                  dd      | j                   j                  dd|      j                  dd      g}| j                   j                  j                  | j	                  |      ddgddgddgddggddgddgddgddggg       | j                   j                  j                  | j	                  |d      ddgddgddgddgddggddgddgddgddgddggg       t        j                  t        d      5  | j	                  |d       d d d        y # 1 sw Y   y xY w)Nr>   r   r   r   )r>   r   r   r   r>   r   r   r   r$  )round_to)r>   r   r   r   r   r   r   r   )r>   r   r   r   r   r   r   r   r   r%  r   r   r!  r#  z'Rounding for padding must at least be 1match)	r   r   r   r	   r  r   r   r   r   )r   r   r   s      r4   test_padr  5  s    
q!5	)366==AU=+KLAFFNN""3771:l/KLFFNN""A!9;S T
 	a%(00A6a%(00A6	A FFNN""
VaVaVaV,VaVaVaV,	
 FFNN""AVaVaVaVaV4VaVaVaVaV4	
 
z)S	T A  s   G//G8c                    | j                  ddgddgddgddgddgg|      }| j                  dd	g      }| j                  j                  j	                  | j                  ||      d
dgddgg       | j                  g d      }| j                  j                  j	                  | j                  ||      d
dgddgddgg       t        j                  t              5  | j                  || j                  j                  g dd             d d d        t        j                  t              5  | j                  || j                  j                  g dd             d d d        y # 1 sw Y   [xY w# 1 sw Y   y xY w)NrX   rY   rZ   r   r   r   r   r   r         "@r   r   r   r   r   r   r   r   r   r   r   r    r   r   )r
  r  r   r   r	   r*   r   r   r   r   r   r   r   r   r  s       r4   test_reduce_sumr  V  s]    	
sc3Z#sc3Z#sDE 	 	A mmQF#GFFNN""q'"c4[3*$=
 mmI&GFFNN""q'"c4[3*sCj$I 
z	" Aq#&&,,|3,?@A 
z	" Cq#&&,,~S,ABC CA AC Cs   0E'.0E3'E03E<c                    t        j                  t        d      5  | j                  | j                  j                  dd|      j                  dd      | j                  j                  ddgd	             d d d        y # 1 sw Y   y xY w)
Nlengths must ber  r>   r#  r   r   r   r   r$   )r   r   r   r  r   r   r   r   r  s     r4   )test_backprop_fails_with_incorrect_lengthr  n  sq     
z);	< 
FFMM!QeM,44Q:FFLL"aL0	

 
 
s   ABB
c                    | j                  ddgddgddgddgd	d
gg|      }| j                  ddg      }| j                  ||      \  }}| j                  j                  j                  || j                  g d             | j                  j                  j                  |ddgddgg       | j                  g d      }t        j                  t        d      5  | j                  ||       d d d        | j                  g d      }t        j                  t        d      5  | j                  ||       d d d        y # 1 sw Y   SxY w# 1 sw Y   y xY w)NrX   r   rY   r   rZ   r   r   r  r   r   r   r   r   r   r   r   r   all sequence lengths must be > 0r  r   r   r>   )lengths must sum up to the number of rows)r
  r  r&   r   r   assert_array_equalr	   r   r   r   r'   r   )r   r   r   r  r  starts_endss         r4   test_reduce_firstr  x  s2    	
sc3Z#sc3Z#tEU 	 	A mmQF#G%%a1NA{FFNN%%k3==3KLFFNN""1SzC:&>?mmI&G	z)L	M $7#$ mmI&G	z)U	V $7#$ $	$ $$ $   E%EE
Ec           	          | j                  ddgddgg|      }| j                  g d      }| j                  ||      }| j                  j                  j                  |ddgddgddgddgddgg       y )NrX   rZ   rY   r   r   r  r   )r
  r  backprop_reduce_firstr   r   r	   )r   r   r  r  r  s        r4   test_backprop_reduce_firstr    s}     
c
S#J/u	=B--	*K		"	"2{	3BFFNN""
c3Z#sc3Z#sc3ZHr7   c                    | j                  ddgddgddgddgd	d
gg|      }| j                  ddg      }| j                  ||      \  }}| j                  j                  j                  || j                  ddg             | j                  j                  j                  |ddgd	d
gg       | j                  g d      }t        j                  t        d      5  | j                  ||       d d d        | j                  g d      }t        j                  t        d      5  | j                  ||       d d d        y # 1 sw Y   SxY w# 1 sw Y   y xY w)NrX   r   rY   r   rZ   r   r   r  r   r   r   r   r   r   r  r  r  r  r  )r
  r  r'   r   r   r  r	   r   r   r   r   )r   r   r   r  r  lastss         r4   test_reduce_lastr    s4    	
sc3Z#sc3Z#tEU 	 	A mmQF#Gq'*HAuFFNN%%eS]]Aq6-BCFFNN""1SzC;&?@mmI&G	z)L	M $7#$ mmI&G	z)U	V $7#$ $	$ $$ $r  c           	          | j                  ddgddgg|      }| j                  ddg      }| j                  ||      }| j                  j                  j                  |ddgddgddgddgddgg       y )	NrX   rZ   rY   r   r   r   r   r   )r
  r  backprop_reduce_lastr   r   r	   )r   r   r  r  r  s        r4   test_backprop_reduce_lastr    s     
c
S#J/u	=BMM1a&!E		!	!"e	,BFFNN""
c3Z#sc3Z#sc3ZHr7   c                    | j                   j                  d|      }|| j                   j                  j                  dd|j                        z  }| j                   j                  g dd      }| j                  ||      \  }}|j                  |k(  sJ | j                   j                  |dk\        sJ | j                   j                  ||j                  d   k        sJ d}t        |      D ]K  \  }}||||z    j                  d      }	| j                   j                  j                  ||   |	       ||z  }M y )	N)r!  r   r   r   r>   )r   r   r   r   r   r   )r   r   rx  r  r   r   r(   r   r   r{   r   r   r	   )
r   r   r   r  maxesr  r  r   r  truths
             r4   test_reduce_max_smr    s    	V5)A		r1agg	..Affll9Cl0G>>!W-LE5;;%66::eqj!!!66::eaggaj()))Ew' 	6%%&.)--1-5&&uQx7r7   c                    | j                   j                  d|      }|| j                   j                  j                  dd|j                        z  }| j                   j                  g dd      }| j                  ||      \  }}|j                  |k(  sJ | j                   j                  |dk\        sJ | j                   j                  ||j                  d   k        sJ d}t        |      D ]K  \  }}||||z    j                  d      }	| j                   j                  j                  ||   |	       ||z  }M t        j                  t              5  | j                  || j                   j                  g d	d             d d d        t        j                  t               5  | j                  || j                   j                  g d
d             d d d        t        j                  t               5  | j                  || j                   j                  g dd             d d d        y # 1 sw Y   xY w# 1 sw Y   gxY w# 1 sw Y   y xY w)N)rC  r   r   r   r>   )r   r   r   r!  r   r   r   r  r  )r   r   r   r   r!  )r   r   rx  r  r   r   r(   r   r   r{   r   r   r	   r   r   r   r   )
r   r   r   r  r  r  r  r   r  r  s
             r4   test_reduce_maxr    s    	WE*A		r1agg	..Affll<sl3G >>!W-LE5;;%66::eqj!!!66::eaggaj()))Ew' 	6%%&.)--1-5&&uQx7
 
z	" Aq#&&,,|3,?@A 
z	" Cq#&&,,~S,ABC 
z	" Dq#&&,,c,BCD DA AC CD Ds$   0H3)0H?:0I3H<?IIc           
         | j                  | j                  j                  dd|      j                  dd      | j                  j	                  g dg dg      j                  d      | j                  j	                  ddgd            }|j                  |k(  sJ | j                  j                  j                  |g d	g d
g dg dg dg       t        j                  t              5  | j                  | j                  j                  ddd      j                  dd      | j                  j	                  g dg dg      j                  d      | j                  j	                  ddgd             d d d        t        j                  t              5  | j                  | j                  j                  dd|      j                  dd      | j                  j	                  g dg dg      j                  d      | j                  j	                  ddgd             d d d        t        j                  t              5  | j                  | j                  j                  dd|      j                  dd      | j                  j	                  g dg dg dg      j                  d      | j                  j	                  g dd             d d d        y # 1 sw Y   fxY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nr>   r#  r   r   r   )r   r>   r   )r>   r   r>   r$   r  )r   rY   r   )rX   r   r   )r   r   r   )r   r   r   r   )r   r   r   r  )backprop_reduce_maxr   r   r   r   r   r   r   r	   r   r   r   r   r  s      r4   test_backprop_reduce_maxr    sW    
	 	 a%(00A6i+,33G<aV7+
B
 88uFFNN""
	
	 
z	" 
FFMM!QcM*221a8FFLL)Y/077@FFLL!QwL/	

 
z	" 
FFMM!QeM,44Q:FFLL)Y/077@FFLL"aL0	

 
z	" 
FFMM!QeM,44Q:FFLL)Y	:;BB7KFFLL'L2	

 

 

 

 
s'   *BKBK&BK2K#&K/2K;c                    | j                  ddgddgddgddgddgg|      }| j                  dd	g      }| j                  j                  j	                  | j                  ||      ddgddgg       | j                  g d
      }| j                  j                  j	                  | j                  ||      ddgddgddgg       | j                  ddgddgddgg|      }| j                  ddg      }| j                  j                  j	                  | j                  ||      ddgddgg       t        j                  t              5  | j                  || j                  j                  ddgd             d d d        t        j                  t              5  | j                  || j                  j                  ddgd             d d d        y # 1 sw Y   [xY w# 1 sw Y   y xY w)NrX   rY   rZ   r   r   r   r   r   r   r  r   r   r   r   r   )r
  r  r   r   r	   r)   r   r   r   r   r   r  s       r4   test_reduce_meanr    s    	
sc3Z#sc3Z#sDE 	 	A mmQF#GFFNN""7#sCj3*%=
 mmI&GFFNN""7#sCj3*sCj%I
 	SzC:Sz:%HAmmQF#GFFNN""7#sCj3*%= 
z	" <366<<Ac<:;< 
z	" =366<<Qs<;<= =< <= =s   0G0GGG'c           
      b   | j                  | j                  j                  dd|      j                  dd      | j                  j	                  ddgd            }|j
                  |k(  sJ | j                  j                  j                  |g dg dg dg dg d	g d	g       t        j                  t        d
      5  | j                  | j                  j                  dd|      j                  dd      | j                  j	                  ddgd             d d d        y # 1 sw Y   y xY w)Nr>   r#  r   r   r   r   r$   )g      ?r=   g      ?)rY   g      @rZ   r  r  r   )backprop_reduce_meanr   r   r   r   r   r   r	   r   r   r   r  s      r4   test_backprop_reduce_meanr  /  s    
	!	!a%(00A6aV7+
B 88uFFNN""
	

 
z);	< 
  FFMM!QeM,44Q:FFLL"aL0	

 
 
s    AD%%D.	reductionc                 N   t        | |      }t        | d|       }| j                  g       } || j                  d|      |      }|dk(  r|\  }} ||||      }n(t        |t              r|\  }}	 |||	      }n	 |||      }|j
                  dk(  sJ |j
                  dk(  sJ y )N	backprop_)r   r    r   r(   rs   r  r   rT  tupler   
r   r   r  funcbackprop_funcr  r  r  r  extras
             r4   test_reduce_empty_batchr  J  s     3	"DC9YK!89MmmBGSYYweY,g6AL 51eW-	Au	51e$1g&77g88wr7   c                 R   t        | |      }t        | d|       }| j                  ddg      } || j                  d|      |      }|dk(  r|\  }} ||||      }n(t        |t              r|\  }}	 |||	      }n	 |||      }|j
                  dk(  sJ |j
                  dk(  sJ y )Nr  r   r   )r   r   r   r(   )r   r   r  r  s
             r4   test_reduce_empty_hiddenr  a  s     3	"DC9YK!89MmmQF#GSYYvUY+W5AL 51eW-	Au	51e$1g&77f88vr7   reduction_raisesc                 h   |\  }}t        | |      }| j                  ddgddgddgddgddgg|      }| j                  g d      }|r,t        j                  t
              5   |||       d d d        y | j                  j                  j                   |||      d	   d
d
g       y # 1 sw Y   y xY w)NrX   rY   rZ   r   r   r   r   r  r>   r   )	rs   r
  r  r   r   r   r   r   r	   )r   r   r  reduction_strr   r  r   r  s           r4   test_reduce_zero_seq_lengthr  x  s     -M6]+I
sc3Z#sc3Z#sDE 	 	A mmI&G]]:& 	"a!	" 	" 	&&yG'<Q'?#sL		" 	"s   "
B((B1c                     | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ y r-   )r   rE   r   r   isnanr   r   r   r  s      r4   	test_mishr    U     	AAA77aggvv||A""$$$$r7   )backprop_clipped_linearbackprop_dishbackprop_gelubackprop_gelu_approxbackprop_hard_sigmoidbackprop_hard_swishbackprop_hard_swish_mobilenetbackprop_hard_tanhbackprop_mishbackprop_relubackprop_relu_kr  backprop_swishc           
      
   t        | |      }t        j                  |      j                  j	                         D cg c]+  }|j
                  t        j                  j                  k(  r|- }}t        |      dk(  rt        j                  t              5   || j                  j                  d|      | j                  j                  d|      | j                  j                  d|             d d d        t        j                  t              5   || j                  j                  d|      | j                  j                  d|      | j                  j                  d|             d d d        y t        j                  t              5   || j                  j                  dd|      | j                  j                  ddd|             d d d        y c c}w # 1 sw Y   xY w# 1 sw Y   y xY w# 1 sw Y   y xY w)Nr   r    r   r   r  r   )rs   rt   ru   rv   rx   ry   r}   r~   r   r   r   r   r   r   r   )r   r   rl   backpropr   positional_argss         r4   .test_eltwise_backprop_rejects_incorrect_shapesr    s   * sBH ""8,77>>@99))/// 	
O 
 ?q ]]:& 	Ru-Qe,Ru-	 ]]:& 	Ru-Ru-Qe,	 	 ]]:& 	c2U3aRu5	 	'	 		 		 	s,   0GAG!AG-AG9!G*-G69Hc                 \   | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ |dk\  j                         |j                  k(  sJ |dk  j                         |j                  k(  sJ y )Nr   r   )r   rc   r   r   r
  r   r   r  r  s      r4   test_relu_kr     s     	AA

1A77aggvv||A""$$$F<<>QVV###H>>qvv%%%r7   c                     | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ y r-   )r   rf   r   r   r
  r   r  s      r4   
test_swishr"    sU     	AA		!A77aggvv||A""$$$$r7   c                 \   | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ |dk\  j                         |j                  k(  sJ |dk  j                         |j                  k(  sJ y )Nr   rX   )r   rd   r   r   r
  r   r   r  r  s      r4   test_hard_sigmoidr$    s     	AAA77aggvv||A""$$$F<<>QVV###H>>qvv%%%r7   c                 \   | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ |dk\  j                         |j                  k(  sJ |dk  j                         |j                  k(  sJ y )NrS  rX   )r   re   r   r   r
  r   r   r  r  s      r4   test_hard_tanhr&    s     	AAaA77aggvv||A""$$$I??&&&H>>qvv%%%r7   c                     | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ y r-   )r   rg   r   r   r
  r   r  s      r4   test_hard_swishr(    sV     	AAqA77aggvv||A""$$$$r7   c                     | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ y r-   )r   rh   r   r   r
  r   r  s      r4   test_hard_swish_mobilenetr*    sX     	AA  #A77aggvv||A""$$$$r7   c                     | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ y r-   )r   rj   r   r   r
  r   r  s      r4   test_gelu_approxr,    sV     	AAA77aggvv||A""$$$$r7   c                     | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ y r-   )r   ri   r   r   r
  r   r  s      r4   	test_dishr.    r  r7   c                     | j                  |      }| j                  |      }|j                  |j                  k(  sJ | j                  j	                  |      j                         rJ y r-   )r   ra   r   r   r
  r   r  s      r4   	test_gelur0    r  r7   c                     | j                  |      }| j                  |j                        }| j                  ||      }|j                  |j                  k(  sJ |dk(  j	                         sJ y )Nr   )r   r   r   r  r   )r   r   r   r  s       r4   test_backprop_mishr2  %  s]     	AAIIaggE			5!	$B88qww!G==??r7   c                    |dk(  rO|dz  |z  |dz  z   |dz  |z  z   |dz  z   }t        d|       D ]"  }||dz  |z  |dz  z   |dz  |z  z   |dz  z   z  }$ nY|dz  |z  |dz  z   |dz  |dz  z  z   |dz  z   }t        d|       D ]%  }||dz  |z  |dz  z   |dz  |dz  z  z   |dz  z   z  }' |dz  }t        ||| |dk\        j                         }|j                  d      j                  |k(  sJ |rT |t        |            }	t        j                  |fd      }
 |t        t        |
j                               |f            }n^t        j                  |fd	      }	t        j                  |fd      }
t        j                  t        |
j                               |f      }t        j                  | |||z  f      }t        j                  | |||z  f      }|	||||
fS )
Nr>   r   r   )depthbir   r$   )r   r   r   r   )r   r   
initialize	get_paramr  r   r   r  intr   r   )r4  dirsnOr|  nIdrawn_paramsrn   lstmr   	size_at_tr   H0C0s                 r4   get_lstm_argsrB  1  s   qyFb=26)BFRK7"q&@q% 	DAQb1f,rAv{:R!VCCH	D Fb=26)BFbAg,>>aGq% 	KAQb1f,rAvq/AABFJJH	KABe	2==?D>>&!&&(222'12 JJj]'B	"C	$8"#=>?XKs3JJj]'B	KK#immo.35	eT2:.	/B	eT2:.	/BBAy))r7   c                      | t        dd            } | t        dd            } | t        dd            |z  } | t        dd            } | t        dd            }t        ||||||       S )Nr>   r   r   rA  r!  )r<  )r   rB  )r<  r4  r9  r:  r|  r;  s         r4   draw_lstm_argsrD  M  sl    !Q EAD	hq"o		%Bhq!n%J	hq"o	Bb*btDDr7   zdepth,dirs,nO,batch_size,nI))r>   r>   r>   r>   r>   )r>   r>   r   r>   r>   )r>   r>   r   r>   r   )r   r>   r>   r>   r>   )r   r>   r   r   r   )r>   r   r   r>   r>   )r   r   r   r   r   c                     t               }t        |||||      \  }}}	}
}|j                  |||	|
|      }| j                  |||	|
|      \  }}t        |d   |d   d   dd       t        |d   |d   d   dd       t        ||d   dd       y Nr   r>   r  r   r  r   )r   rB  lstm_forward_trainingr	   )r   r4  r9  r:  r|  r;  reference_opsr   r@  rA  r   r?  	referencer  	fwd_states                  r4   test_lstm_forward_trainingrK  W  s     EM#0b*b#Q FBAy33FBAyQI,,VRQ	JLAyIaL)A,q/4HIaL)A,q/4HAy|$T:r7   zFlaky, skip temporarily)argsc                    |\  }}}}}t               }|j                  |||||      }| j                  |||||      \  }	}
t        |
d   |d   d   dd       t        |
d   |d   d   dd       t        |	|d   dd       y rF  )r   rG  r	   )r   rL  r   r@  rA  r   r?  rH  rI  r  rJ  s              r4   test_lstm_forward_training_fuzzrN  n  s    
 $( FBAyEM33FBAyQI,,VRQ	JLAyIaL)A,q/4HIaL)A,q/4HAy|$T:r7   c                  N   t        t        d      t              sJ t        t        d      t              sJ 	 ddlm}  t        t        d      |       sJ 	 	 ddlm} t        t        d      |      sJ 	 t        j                  t              5  t        d       d d d        t        t              }|j                  t        k(  sJ y # t        $ r t        t        d      t              sJ Y w xY w# t        $ r t        t        d      t              sJ Y w xY w# 1 sw Y   xY w)Nr   cupyr   )AppleOpscpu)BigEndianOpsblah)rT  r   r   r   thinc_apple_opsrQ  ImportErrorthinc_bigendian_opsrS  r   r   r   r   r   r   )rQ  rS  r   s      r4   test_get_opsrX  |  s    gg&111gfow///4,'%.(333
44'%.,777 
z	" 
e*C66U??  4'%.(3334  4'%.(3334 s/   C C1 D$C.-C.1$DDD$c                  R   t               } t        d      5  t               }|j                  dk(  sJ 	 d d d        t        d      5  t               }|j                  dk(  sJ 	 d d d        t               }| j                  |j                  k(  sJ y # 1 sw Y   _xY w# 1 sw Y   ;xY w)Nr   rP  )r   r   name)	class_opsnew_opss     r4   test_use_opsr]    s    !I		 '!#||w&&&' 
 &!#||v%%%& G>>W\\)))' '& &s   BBBB&c                  :   t        d       t               } g d}| j                  d|      }t        |      g dg dgk(  sJ | j                  d dD        |      }t        |      g ddd	gd
ggk(  sJ t        | j                  dt	        j
                  |                  }t        |d   t        j                        sJ t	        j                  |d   t	        j
                  g d            sJ t	        j                  |d   t	        j
                  g d            sJ t        | j                  d dD        |d            }|g ddd	gd
ggk7  sJ t        |d         dk(  sJ t        |d         dk(  sJ t        |d         dk(  sJ t        j                  t              5  | j                  dd t        d      D               d d d        t        j                  t              5  | j                  dd       d d d        y # 1 sw Y   >xY w# 1 sw Y   y xY w)Nr   )r>   r   r   r   r   r!  r   )r>   r   r   )r   r   r!  c              3       K   | ]  }|  y wr-   rK   .0r   s     r4   	<genexpr>z!test_minibatch.<locals>.<genexpr>  s     21Q2   r  r   r   r!  r>   c              3       K   | ]  }|  y wr-   rK   r`  s     r4   rb  z!test_minibatch.<locals>.<genexpr>  s     !7!!7rc  Tshuffler   r    c              3       K   | ]  }|  y wr-   rK   r`  s     r4   rb  z!test_minibatch.<locals>.<genexpr>  s     111rc  r   )r   r   	minibatchlistr   r   rT  rU  r  r   r   r   r   r   )r   itemsbatchess      r4   test_minibatchrl    s   A

CEmmAu%G=Y	2222mm2	2E:G=YA44443==EMM%$89:Ggaj%--000WQZy)ABBBWQZy)ABBB3==!7Y!7=MNGy1a&1#....wqz?awqz?awqz?a	z	" 3b1eCj123	z	"  b$   3 3   s   %#H)HHHc            	         t        d       t               } t        j                  g d      }t        j                  g d      }t	        | j                  d||            }t        j                  |      j                         ddgddgdd	gd
dggk(  sJ t	        | j                  d||d            }t        |      dk(  sJ t        |d         dk(  sJ t        |d         dk(  sJ t	        | j                  dg dg d            }|ddgddggdd	gd
dgggk(  sJ t        j                  t              5  | j                  dd t        d      D        d t        d      D               d d d        t        j                  t              5  | j                  d|d t        d      D        |       d d d        y # 1 sw Y   PxY w# 1 sw Y   y xY w)Nr   r  )r   r!  r#  r$  r   r>   r   r!  r   r   r#  r$  Tre  r    c              3       K   | ]  }|  y wr-   rK   r`  s     r4   rb  z"test_multibatch.<locals>.<genexpr>  s     2!A2rc  r   c              3       K   | ]  }|  y wr-   rK   r`  s     r4   rb  z"test_multibatch.<locals>.<genexpr>  s     4K1Q4Krc  c              3       K   | ]  }|  y wr-   rK   r`  s     r4   rb  z"test_multibatch.<locals>.<genexpr>  s     !8!!8rc  )r   r   r   r   ri  
multibatchconcatenatetolistr   r   r   r   r   )r   arr1arr2rk  s       r4   test_multibatchrv    s   A

C==&D==&D3>>!T401GW%,,.Aq6Aq6Aq6Aq62RRRR3>>!T4>>?Gw<1wqz?awqz?a3>>!\<@AGAA'1a&1a&)9::::	z	" Mr2uSz24Kc
4KLM	z	" @r4!8U3Z!8$?@ @M M@ @s   64F9%G9GGc                     t               } t        j                  g dt        j                        }t	        dd      D ]?  }t        | j                  ||            t        d|j                  d   |dz
  z
        k(  r?J  t        | j                  d|            dk(  sJ t        | j                  |j                  d   dz   |            dk(  sJ y )N)r>   r   r   r   r   r   r>   r    r   r   )	r   r   r   r   r   r   ngramsr   r   )r   rt  ns      r4   test_ngramsrz    s    

C===D1b\ K3::a&'3q$**Q-1q52I+JJJJKszz"d#$)))szz$**Q-!+T23q888r7   z1.9.0zneeds PyTorch 1.9.0
torch_funcir5  )	min_value	max_valuer   )r2   r  c           	         dd l }|\  }}t        | |      }t        | d|z         }	| j                  |g|      }
t        |
d      } ||      } ||
      }|j	                          |
j
                  |j
                  k(  sJ ||
usJ  ||
d      }||
u sJ | j                  j                  ||d      sJ | j                  j                  ||j                         d	      sJ | j                  |g|      }
| j                  |g|      }|j                         }t        j                  |	      }|j                  D ch c]	  }|d
v s| }}|h dk(  r |	|||
      }|j
                  |
j
                  k(  sJ ||usJ  |	|||
d      }||u sJ | j                  j                  ||      sJ | j                  j                  |j                  j                         |z  t        |      d      sJ y |ddhk(  r |	||      }|j
                  |
j
                  k(  sJ | j                  j                  | |	||d            sJ | j                  j                  |j                  j                         |z  t        |      d      sJ y |ddhk(  r |	||
      }|j
                  |
j
                  k(  sJ | j                  j                  | |	||
d            sJ | j                  j                  |j                  j                         |z  t         |	||
            d      sJ y t!        d|       c c}w )Nr   r  r   Tr  r  rc  r   r   )r  r   r  >   r   r  r  )r  r   )r  r  r   r  r  r  )r  )r  r  r  r   r   )r  r   r  z5No PyTorch comparison implemented for parameter set: )r3   rs   r   r   r  r   r   isclosedetachcopyrt   ru   rv   r  itemfloatNotImplementedError)r   r   r2   r  r{  r3   	func_namepytorch_funcforwardr  x_thincx_torchr  y_thincy_think_inplacedY_thincdY_thinc_inplacesr   r   dx_thincdx_thinc_inplaces                         r4   !test_compare_activations_to_torchr    s@    (I|c9%GsK)34H kk1#Uk+Gwd3GWAgGJJL==GMM)))'!!!gt4Og%%%66>>'?>???66>>'188:E>:::kk1#Uk+G{{B4u{-H}}(#A?A.>)>a?F?!!H7;~~...x'''#7gt
  #3333vv~~h(8999vv~~gll//1B6he~TTT	C;	H0~~...vv~~(GTB
 	
 
 vv~~gll//1B6he~TTT	D#;	H0~~...vv~~h"2gtL
 	
 
 vv~~LL"$eHX,I&JQV  
 	
 
 "CF8L
 	
= @s    	L>
L>r  rU   c                    | j                   j                  |g      }| j                   j                  | j                  |d      | j	                  |            sJ | j                   j                  | j                  | j                  dg      |d      | j                  | j                  dg      |            sJ | j                   j                  | j                  |dd      | j                  |            sJ | j                   j                  | j                  | j                  dg      |dd      | j                  | j                  dg      |            sJ y )Nr   )max_valrX   r<   r=   )slopeoffset)
r   r   r  clipped_linearrc   r  	asarray1fr  rd   r  )r   r2   r  s      r4   test_clipped_linearr    s<    ffnnaS!G66>>#,,Wc,BCJJwDWXXX66>>##CMM3%$8'3#OCMM3%0':   66>>7#c:C<L<LW<U   66>>##MM3% 'S 	$ 	
 	!!#--"6@	  r7   
byte_order)><=|c                    | j                   j                  |g      }| j                  ||      }t        j                  | j                  |      | j                  |            sJ |dv r:|j
                  j                  d      j                  d      j                  |k(  sJ y |j
                  j                  |j
                  j                  k(  sJ y )N)r  )r  r  S)r   r   to_numpyr   r  r   newbyteorder	byteorder)r   r  r2   r  s       r4   test_to_numpy_byteorderr  .  s    
 	sAQ:.AS\\!_cll1o>>>Zww##C(55c:DD
RRRww  AGG$5$5555r7   c                  d    t         D ]  } t        j                  |       }|J  t               J y r-   )r   r   get_functionr   )kernel_namecompiled_kernels     r4   test_custom_kernel_compilationr  =  s=    # +!..{;***+ =$$$r7   c                 N    ddg}|t        | j                  |d            k(  sJ y )NrA  l   4dU5
 r   r   )ri  r   )r   uint64_lists     r4   test_asarray_from_list_uint64r  F  s/     +,K$s{{;h{GHHHHr7   r-   )rt   platformtypingr   r   r   r   
hypothesisr   r   hypothesis.strategiesr   r   numpy.testingr	   packaging.versionr
   	thinc.apir   r   r   r   r   r   r   r   thinc.backends._custom_kernelsr   r   r   thinc.compatr   r   r   thinc.typesr   
thinc.utilr   r    r   r   r   MAX_EXAMPLESVANILLA_OPS	NUMPY_OPSBLIS_OPSCPU_OPSXP_OPSappendALL_OPSFLOAT_TYPES	INT_TYPES
REDUCTIONSREDUCE_ZERO_LENGTH_RAISESrk   TORCH_FUNCSmarkparametrizer   filterwarningsr   r   r   r   r   r   r   r   r   
arrays_BOPr  r  r  r  r?  rJ  rO  rQ  rX  r^  rh  rn  rp  rr  skipifr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r"  r$  r&  r(  r*  r,  r.  r0  r2  rB  rD  rK  skiprN  rX  r]  rl  rv  rz  floatsr  r  r  r  r  rK   r7   r4   <module>r     s        & 5 ) %	 	 	 N M ? ?   )  5%jJ	T"k
"

MM')
K=
 )$g	W

 @F &(KK '23 4: (459 6 )9 (" )", '# (# '' (' ' ( (+.((;< = / ) '!Z!!#$ % ( (' )' (. ). (+.	|d3				 ! " 4 / )& (+.
 / )
$ (	|d3				 N ! 4 )N '+.( / ((V '+.+ / (+\ '+.? / (?D '+. / (& '+. / (& '+.; / (; (+.	|d3				 N ! 4 / )N& '+. / (* '+.8 / (8 '+. / (( '+.J / (J6 $-=>1v&* ' ?*( $-=>1v&. ' ?.. (+.	|d3				  ! 4 / )  (	|d3				 . ! 4 ). (	|d3				 / ! 4 )/,1
8X$ 	M/:(8
7 9 ) ;
7 $8g$8x$89+ :+$ $8g$8x$895 :5 G,	|d3				   ! 4 -  (+	"9: ; )> (+.C / )C, (+.
 / )
 (+.$ / )$$ (+. / ) (+.$ / )$$ (+. / ) (+. / )  (+.D / )D8 (+.%
 / )%
P (+.= / )=: (+.
 / )
2 (+.j1 2 / )( (+.j1 2 / )( (+.+-FGM H / )M  (	|d3				 % ! 4 )% '+.$% / ((8 (	|d3				 & ! 4 )& (	|d3				 % ! 4 )% (	|d3				 & ! 4 )& (	|d3				 & ! 4 )& (	|d3				 % ! 4 )% (	|d3				 % ! 4 )% (	|d3				 % ! 4 )% (	|d3				 % ! 4 )% (	|d3				 % ! 4 )% (	|d3				  ! 4 )*8 E E '!; (; 23'	|d3N;  4 ( 4;2	* .@&9 	M/:MGG$44=RS(9i"89{3	|d3j#4z238
	 4 4 : ) T ;8
v (	|d3			SB	78 9 4 )$ (';<	|d3			SB	786 9 4 = )6 $-=>% ?% (I )Ir7   