
    i                        d dl Z d dlZd dlZd dlZd dlmZmZ d dlZd dlm	Z	 d dl
mZmZ d dlmZmZmZmZmZmZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZm Z m!Z!m"Z" ddl#m$Z$ ejJ                  d        Z& ejJ                         d        Z'ejJ                  d        Z(ejR                  jU                  d      d        Z+ejR                  jU                  d      d        Z,dZ-ejR                  jU                  d      d        Z.d Z/d Z0d Z1d Z2d Z3d Z4d Z5d Z6d Z7ejR                  jq                  d       d!        Z9d" Z:d# Z;d$ Z<d% Z=d& Z>d' Z?d( Z@d) ZAd* ZBd+ ZCd, ZDd- ZEd. ZFd/ ZGd0 ZHd1 ZId2 ZJejR                  j                  d3g d4d5d6gd gd gd7ggd d7gdggffg d8d9d6gd gd gd gd7ggg d:d;ggffd<d=gd5d6gd gd d7ggd d7gd7ggffg d>g d?d d7gdgdggd gd gd7dggffg d?g d@d gd7gdd;ggd gd7gdgdggffdAd<gd<gg d ggd7ggffg dBg dCd gd d7gd7gdggd d7gd7dgd;ggffg      dD        ZLdE ZMdF ZNdG ZOdH ZPdI ZQdJ ZRdK ZSdL ZTdM ZUdN ZVdO ZWdP ZXdQ ZYdR ZZdS Z[dT Z\y)U    N)Adamcompounding)English)DocDocBin)	AlignmentCorpusExamplebiluo_tags_to_offsetsbiluo_tags_to_spansdocs_to_jsoniob_to_biluooffsets_to_biluo_tagsget_alignments)AlignmentArray)json_to_docs)train_while_improving)get_words_and_spacesload_config_from_strload_model_from_path	minibatch   )make_tempdirc                      t               } g d}g d}g d}g d}g d}g d}g d}dgt        |      z  }d	|d
<   d|d<   d|d<   d|d<   d|d<   ddd}	t        | j                  ||||||||	      }
|	|
_        |
S )N
Sarah'ssisterflewtoSiliconValleyviaLondon.)
NNPPOSNNVBDINr'   r'   r+   r'   r&   )
PROPNPARTNOUNVERBADPr,   r,   r0   r,   PUNCT)
NounType=prop|Number=singzPoss=yeszNumber=singzTense=past|VerbForm=fin r2   r2   r3   r2   zPunctType=peri)
r   r      r4   r4         r4         )
posscasensubjROOTprepcompoundpobjr=   r?   punct)
r   r   r   flyr!   r"   r#   r$   r%   r&   OB-PERSONr   zI-PERSON   B-LOCr8   I-LOCr5   zB-GPE         ?        )TRAVELBAKING)wordstagsposmorphsheadsdepslemmasents)r   lenr   vocabcats)nlprL   rM   rN   rO   rP   rQ   rR   rS   rV   docs              s/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/spacy/tests/training/test_training.pyrX   rX   #   s    
)C^ELD
]C=F +EaD^F53u:DDGDGDGDGDGS)D
		
C CHJ    c                  &    g dg dg dg dg ddS )N)rD   r   r4   r6   r8   r5   r7   )HithereeveryoneItisjustme)TTTTTTF)INTJADVPRONre   AUXrd   re   )rD   r   r   rD   r   r   r   )idsrL   spacesrM   sent_starts rj   rZ   rY   merged_dictrk   H   s     %F=E, rZ   c                  .    t               } | j                  S N)r   rU   )rW   s    rY   rU   rU   S   s    
)C99rZ   i  c            
      &   dg gdg gdg gdg gdg gdg gddggd	d
ggddggg	} t               }|j                  d      }| D ]!  \  }}|D ]  \  }}}|j                  |        # |j                          t	        d      D ]Z  }t        j                  |        | D ]>  \  }	}
t        j                  |j                  |	      d|
i      }|j                  |g       @ \ t               5 }|j                  |       t        |      }ddd       | D ]s  \  }	}
 |	      }|j                  D ci c]%  }|j                  |j                   f|j"                  ' }}|
D ]'  \  }}}||f|v r|||f   |k(  sJ  e|
st%        |       u y# 1 sw Y   xY wc c}w )a$  Test that adding entities and resuming training works passably OK.
    There are two issues here:
    1) We have to re-add labels. This isn't very nice.
    2) There's no way to set the learning rate for the weight update, so we
        end up out-of-scale, causing it to learn too fast.
    heyhowdyz	hey therehellohizi'm looking for a place to eatz,i'm looking for a place in the north of town)   $   LOCATIONzshow me chinese restaurants)rG      CUISINEzshow me chines restaurants)rG      rw   ner   entitiesN)r   add_pipe	add_label
initializerangerandomshuffler
   	from_dictmake_docupdater   to_diskr   rS   
start_charend_charlabel_	Exception)
TRAIN_DATArW   ry   _offsetsstartendlabelitnraw_textentity_offsetsexample	model_dirnlp2rX   entrS   s                    rY   test_issue999r   Y   s    
	"	b	"	r
	)2.	7:N9OP	&);(<=	%(:';<
J )C
,,u
C  !
7!( 	!E3MM% 	!! NNRy "z"(2 	"$Hn''X&^(DG JJy!		"" 
 /9I#I./ %/ 	* .8nFIhhOs.

:OO!/ 	*E3s|t#UCL)U222!#D/)	*	*	/ / Ps   $F'*FFi2  c                     dddddddddddd	ddd
dddddddddddddddddddddddg	g ddddddd	ddddddddddddddd d!ddd"d#ddd$dddd%dddd&dddd'dddd(d)ddgg dgd*d+d,d-d.d,gd/d0dd1dddd2dddd3ddd
d4dddd5dddd6dddd7dddd8dddd9ddddddg
g dddddgg dgd*d.d,d-d+d,gd/gd:} t               }g d;}t               5 }|d<z  }t        | g      }t        ||=      j	                         }|j                  d>      5 }|j                  |       d d d        t        |      }t         ||            }	t        |	      dk(  sJ g }
|	D ]!  }|
j                  |j                                # t        |
      dk(  sJ 	 d d d        y # 1 sw Y   wxY w# 1 sw Y   y xY w)?Nr   zRHow should I cook bacon in an oven?
I've heard of people cooking bacon in an oven.HowrB   )idorthry   rD   shouldr   Ir4   cookr6   baconr8   inr5   anr7   ovenrG   ?)tokensbrackets	   

      z've   heard   ofrx   peoplerv   cooking            rz   r&   bakingrH   )r   value
not_bakingrI   )raw	sentencesrV   z5What is the difference between white and brown eggs?
Whatr`   the
differencebetweenwhiteandbrowneggsr   
paragraphs)ORTH
SENT_STARTENT_IOBENT_TYPEztest4402.spacy)docsattrswb)r   r   r   r   to_bytesopenwriter	   listrT   extendsplit_sents)	json_datarW   r   tmpdiroutput_filer   datafile_reader
train_datasplit_train_dataegs               rY   test_issue4402r      s     m $%eC@#$hsC#$c#>#$fSA#$gcB#$d3?#$d3?#$fSA#$c#>
# %'  $%d3?#%s3?#%uSA#%wsC#%tC@#%xD#%yE#%wsC#%tC@#%tC@#%vcB#%s3?# %' D '5*S9G'R P $%fSA#$d3?#$eC@#$l3G#$iD#$gcB#$eC@#$gcB#$fSA#$c#># %' (*4 DESUV!& '5*S9)SB
EIL )C9E	 *6//YK(4u-668d# 	uKK	$&+&
:!### 	6B##BNN$45	6#$)))* *	 		* *s%   (>F:&F.8A,F:.F7	3F::Ga  
[nlp]
lang = "en"
pipeline = ["tok2vec", "tagger"]

[components]

[components.tok2vec]
factory = "tok2vec"

[components.tok2vec.model]
@architectures = "spacy.Tok2Vec.v1"

[components.tok2vec.model.embed]
@architectures = "spacy.MultiHashEmbed.v1"
width = ${components.tok2vec.model.encode:width}
attrs = ["NORM","PREFIX","SUFFIX","SHAPE"]
rows = [5000,2500,2500,2500]
include_static_vectors = false

[components.tok2vec.model.encode]
@architectures = "spacy.MaxoutWindowEncoder.v1"
width = 96
depth = 4
window_size = 1
maxout_pieces = 3

[components.tagger]
factory = "tagger"

[components.tagger.model]
@architectures = "spacy.Tagger.v2"
nO = null

[components.tagger.model.tok2vec]
@architectures = "spacy.Tok2VecListener.v1"
width = ${components.tok2vec.model.encode:width}
upstream = "*"
iu  c            	      x  
 ddg difddg difg} t        j                  t        t                    }g 
| D ]<  }
j	                  t        j                  |j                  |d         |d                > |j                  
fd	      }t        d
      D ]  }i }|j                  
||        g d}t        |j                  |d            }t        |j                  |d            }|dd D 	cg c]  }	|	d   j                   c}	|dd D 	cg c]  }	|	d   j                   c}	k(  sJ yc c}	w c c}	w )z<Test that an empty document doesn't mess up an entire batch.zI like green eggsrM   )NVJr   zEat blue ham)r   r   r   r   rD   c                       S rm   rj   )train_exampless   rY   <lambda>z test_issue7029.<locals>.<lambda>  s    N rZ   )get_examples2   sgdlosses)firstsecondthirdfourthr   thensomer3   )
batch_sizer6   N)r   from_configr   CONFIG_7029appendr
   r   r   r~   r   r   r   pipetag_)r   rW   t	optimizerir   textsdocs1docs2rX   r   s             @rY   test_issue7029r     s<    
v';<=	&/23J 

2;?
@CN Kg//QqT0BAaDIJK,BCI2Y A

>y
@A NE%A./E%A./E#(":.CCFKK.%PSQS*2U33q6;;2UUUU.2Us   5D2D7c                     g d}g d}t        | ||      }t        d      t        d      dfg}t        ||      }|g dk(  sJ y )N)r   r    r!   r%   r&   )TTTFTrL   rh   
I flew to zI flew to LondonLOCrB   rB   rB   U-LOCrB   r   rT   r   en_vocabrL   rh   rX   r{   rM   s         rY   test_gold_biluo_Ur  "  sO    .E,F
heF
3C\"C(:$;UCDH h/D0000rZ   c                     g d}g d}t        | ||      }t        d      t        d      dfg}t        ||      }|g dk(  sJ y )N)r   r    r!   San	Franciscor&   )TTTTFTr   r   I flew to San Franciscor   )rB   rB   rB   rE   L-LOCrB   r   r  s         rY   test_gold_biluo_BLr	  +  sO    8E2F
heF
3C\"C(A$BEJKH h/D9999rZ   c                     g d}g d}t        | ||      }t        d      t        d      dfg}t        ||      }|g dk(  sJ y )Nr   r    r!   r  r  r#   r&   TTTTTFTr   r   I flew to San Francisco Valleyr   )rB   rB   rB   rE   rF   r  rB   r   r  s         rY   test_gold_biluo_BILr  4  sO    BE8F
heF
3C\"C(H$I5QRH h/DBBBBrZ   c                     g d}g d}t        | ||      }t        d      t        d      dft        d      t        d      dfg}t        j                  t              5  t        ||       d d d        y # 1 sw Y   y xY w)Nr  r  r   r   r  r   r  )r   rT   pytestraises
ValueErrorr   )r  rL   rh   rX   r{   s        rY   test_gold_biluo_overlapr  =  sx    BE8F
heF
3C	\	C @A5I	\	C 9:EBH 
z	" -c8,- - -s   A44A=c                     g d}g d}t        | ||      }t        d      t        d      dfg}t        j                  t              5  t        ||      }d d d        g dk(  sJ y # 1 sw Y   xY w)N)r   r    r!   r  r  zValley.)TTTTTFr   r   r  r   )rB   rB   rB   -r  r  )r   rT   r  warnsUserWarningr   r  s         rY   test_gold_biluo_misalignr  I  sp    >E2F
heF
3C\"C(H$I5QRH	k	" 4$S(3411114 4s   A''A0c                 @   g d}g d}|D cg c]  }| j                   j                  |       }}t        | |      }t        | |      }|j                  dt	        j
                  |d            }t        ||      }|j                  dd      }|g dk(  sJ y c c}w )	Nr   likestuffr.   r/   r.   rL   TAGuint64)dtypeT	as_string)stringsaddr   
from_arraynumpyarrayr
   get_aligned)r  rL   rM   tagtag_ids	predicted	referencer   s           rY   test_example_constructorr.  S  s    "E#D489Sx##C(9G9HE*IHE*I$$UEKKx,PQIi+Gu5D++++ :s   "Bc                     g d}g d}t        | |      }t        j                  |d|i      }|j                  dd      }|g dk(  sJ y )Nr  r  r  TAGSr  Tr"  )r   r
   r   r)  )r  rL   rM   r,  r   s        rY   test_example_from_dict_tagsr1  _  sR    "E#DHE*I	FD>:Gu5D++++rZ   c                     g d}g d}t        | ||      }t        j                  |d|i      }|j                         }|g dk(  sJ y )NabcdTTFTr   rL   )NNNNr   r
   r   get_aligned_nerr  rL   rh   r,  r   ner_tagss         rY   test_example_from_dict_no_nerr=  h  sN     E&FHE&9I	GU+;<G&&(H////rZ   c                     g d}g d}t        | ||      }t        j                  ||g dd      }|j                         }|g dk(  sJ y )Nr3  r8  r   )r   NNNrL   r{   r9  r;  s         rY   test_example_from_dict_some_nerr@  q  sT     E&FHE&9IU0KLG &&(H2222rZ   zignore::UserWarningc                     dddddddddddd	dd
ddddddddddddddgigigdg}t        t        |            }t        |      dk(  sJ |D ]  }|j                  d      sJ  D ]  }|j                  dk(  rJ  t        t        |j                  |D cg c]  }|j                   c}|D cg c]  }t        |j                         c}      |      }|j                         }|g dk(  sJ y c c}w c c}w )NrD   r   r   nnr'   zMs.)depheadr*  r   r;   Haagr<   r   VBZplaysdobjr   Eliantir@   r&   r   r   r   )NNNNN)r   r   rT   has_annotationent_iobr
   r   rU   textboolwhitespace_r:  )r  r   r   rX   tokenwr   r<  s           rY   test_json_to_docs_no_nerrR  |  sZ      $(,au U+2,-+0,2	!" ,2,-+0,3	!" ,2,.+0,5	!" )0CQT U)'" 	
"DF T"#Dt9>> 1%%i0001 "}}!!!"	II#&'a166'145AD'5	

 	
B !!#H5555 (5s   C63C;c                 0   g d}g d}g d}t        | |      }t        j                  |||d      }|j                  dk(  sJ |j	                         }t        |      dk(  sJ |d   j                  d	k(  sJ |d
   j                  dk(  sJ g d}g d}g d}t        | |      }t        j                  |||d      }|j                  dk(  sJ |j	                         }t        |      dk(  sJ |d   j                  d	k(  sJ |d
   j                  dk(  sJ y )N)r   r    r!   San Francisco Valleyhadzloads of fun)
r   r    r!   r  r  r#   rU  loadsr   fun)
TFFFFFTFFFr  )rL   ri   z0I flew to San Francisco Valley had loads of fun r   r   zI flew to San Francisco Valley rD   zhad loads of fun )	r   r    r!   r  r  r#   rU  rV  zof fun)r   r    r!   San Franciscor#   rU  zloads ofrW  )TFFFFTFF)r   r
   r   rM  r   rT   )r  rL   
gold_wordsri   rX   r   split_exampless          rY   test_split_sentencesr[    s6   NE_JVK
he
$Cz+%VWG<<MMMM((*N~!###!!!%FFFF!!!%8888WEYJHK
he
$Cz+%VWG<<MMMM((*N~!###!!!%FFFF!!!%8888rZ   c                 r   g d}g d}t        | ||      }d}t        |      t        |dz         dfg}g d}t        j                  |||d      }|j	                         }	|	g d	k(  sJ t        d
      t        d      dft        |      t        |dz         dfg}g d}t        j                  |||d      }|j	                         }	|	g dk(  sJ t        d
      t        d      dft        |      t        |dz         dfg}g d}t        j                  |||d      }|j	                         }	|	g dk(  sJ y )N)Mr and 	Mrs Smithflew torT  r&   TTTFFr   Mr and Mrs Smith flew to rT  r   )Mr and Mrs Smithr    r!   r  r  r#   r&   r?  r   r]  rb  PERSON	Mr andMrsSmithr    r!   r  r  r#   r&   )rB   U-PERSONrB   r   rB   
Mr and Mrs)rB   NrB   r   rB   r   rT   r
   r   r:  
r  en_tokenizerrL   rh   rX   prefixr{   rY  r   r<  s
             rY   test_gold_biluo_one_to_manyrn    sY   LE-F
heF
3C(FVc&+A"ABEJKHVJzx%PQG&&(H4444 
Y/0(;	Vc&#99:EBH
 ]Jzx%PQG&&(H;;;; 
Y\*H5	Vc&#99:EBH
 ]Jzx%PQG&&(H5555rZ   c                    g d}g d}t        | ||      }d}t        |      t        |dz         dfg}g d}t        j                  |||d      }|j	                         }	|	g d	k(  sJ t        d
      t        d      dft        |      t        |dz         dfg}g d}t        j                  |||d      }|j	                         }	g d}
|	|
k(  sJ y )Nrd  )	TTTTTTTFFr   ra  rT  r   )rb  r_  rT  r&   r?  )	rB   rB   rB   rB   rB   rE   rF   r  rB   r]  rb  rc  )re  r^  r_  rT  r&   )	rB   rC   zL-PERSONrB   rB   rE   rF   r  rB   rj  )r  rl  rL   rh   rX   rm  r{   rY  r   r<  expecteds              rY   test_gold_biluo_many_to_onerq    s    WEEF
heF
3C(FVc&+A"ABEJKHMJzx%PQG&&(HPPPP 
Y/0(;	Vc&#99:EBH QJzx%PQG&&(HVHxrZ   c                    g d}g d}t        | ||      }d}t        |      t        |dz         dfg}g d}t        j                  |||d      }|j	                         }	|	g d	k(  sJ t        d
      t        d      dft        |      t        |dz         dfg}g d}t        j                  |||d      }|j	                         }	|	g dk(  sJ y )N)ri  rg  r    r!   rX  r#   r&   )TTTTTFFr   ra  rT  r   )Mrzand Mrs Smithr_  r  Francisco Valleyr&   r?  rB   rB   rB   rB   rE   r  rB   r]  rb  rc  )re  r^  r_  r  rt  r&   )NNrB   rB   rE   r  rB   rj  rk  s
             rY   test_gold_biluo_misalignedrv    s    QE9F
heF
3C(FVc&+A"ABEJKHSJzx%PQG&&(HBBBB 
Y/0(;	Vc&#99:EBH TJzx%PQG&&(HDDDDrZ   c                     t        g dd      \  }}t        | ||      }d}t        |      t        |dz         dfg}g d}g d}t        j                  ||||d	      }	|	j                         }
|
g d
k(  sJ y )N)r   r    r!   rX  r#   r&   z I flew  to San Francisco Valley.r   zI flew  to rT  r   )r   r     r!   rT  r&   )TTFTFFrL   rh   r{   ru  )r   r   rT   r
   r   r:  )r  rl  rL   rh   rX   rm  r{   rY  gold_spacesr   r<  s              rY   %test_gold_biluo_additional_whitespacer{    s    (;*ME6 heF
3CFVc&+A"ABEJKHFJ9Kz[hOG &&(HBBBBrZ   c                     |d      }g d}g d}dg}t        j                  ||||d      }|j                         }|g dk(  sJ  |d      }g d}g d}dg}t        j                  ||||d      }|j                         }|g d	k(  sJ y )
NzI'll return the A54 amount)r   'llreturnr   A54amount)FTTTFTF)r   r   MONEYry  )rB   rB   rB   rB   zU-MONEYrB   zI'll return the $54 amount)r   r}  r~  r   $r  r  )rB   rB   rB   rB   zB-MONEYzL-MONEYrB   )r
   r   r:  )r  rl  rX   rY  rz  r{   r   r<  s           rY   test_gold_biluo_4791r  -  s    
3
4CCJ?K!"Hz[hOG &&(H;;;;
3
4CCJ?K!"Hz[hOG &&(HFFFFrZ   c                     d}g d}ddg} | |      }t        ||      }||k(  sJ t        ||      }|D cg c]
  }|d   s	| }}||k(  sJ y c c}w )N$I flew to Silicon Valley via London.rB   rB   rB   rE   r  rB   U-GPErB   )r      r   )   #   GPEr   )r   r   )rl  rM  
biluo_tagsr   rX   biluo_tags_convertedoffsets_convertedr   s           rY   'test_roundtrip_offsets_biluo_conversionr  C  su    1DEJ0G
t
C0g>:----c:>(/:3q6::''' ;s   
AAc                 6    | d      }g d}t        ||      }|D cg c]  }|j                  s| }}t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  d	k(  sJ y c c}w )
Nr  r  r   r   zSilicon Valleyr   rD   r%   r  )r   r   rT   rM  )rl  rX   r  spansspans        rY   test_biluo_spansr  O  s    
=
>CEJZ0E#3dt{{T3E3u:??8==,,,,8??e###8==H$$$8??e### 4s
   BBc                    g d}g d}t        | ||      }d}dt        d      dft        |      t        |dz         d	fg}g d
}t        j                  |||d      }|j                  j
                  }	|	D 
cg c]  }
|
j                  |
j                  f c}
ddgk(  sJ |j                  |	      }|D 
cg c]  }
|
j                  |
j                  f c}
ddgk(  sJ y c c}
w c c}
w )N)rb  r    r!   rT  r&   r`  r   ra  r   rb  rc  rT  r   )
rs  r   rf  rg  r    r!   r  r  r#   r&   r?  r   r6   r5   r   )r   rD   )r4   r6   )	r   rT   r
   r   r-  rS   r   r   get_aligned_spans_y2x)r  rl  rL   rh   rX   rm  r{   
tokens_refr   ents_refr   ents_y2xs               rY   test_aligned_spans_y2xr  [  s    KE-F
heF
3C(F	
C"#X.	Vc&#99:EBH
 `Jzx%PQG  %%H,45SSYY 5&&9IIII,,X6H,45SSYY 5&&9IIII 65s   8C 6C%c                 ,   d}t               }ddddddg}|j                  d      }|j                  |        ||      }|j                  D cg c]  }|j                  |j
                  f c}dd	gk(  sJ d
}dt        d      dft        |      t        |dz         dfg}	g d}
t        j                  ||
|	d      }|j                  j                  D cg c]  }|j                  |j
                  f c}ddgk(  sJ |j                  j                  }|D cg c]  }|j                  |j
                  f c}dd	gk(  sJ |j                  |      }|D cg c]  }|j                  |j
                  f c}ddgk(  sJ y c c}w c c}w c c}w c c}w )Nz-Mr and Mrs Smith flew to San Francisco Valleyrc  rb  )r   patternr   rT  entity_rulerr  r  ra  r   )ri  rg  r    r!   rX  r#   r?  )r   r   )r6   r5   )r   r|   add_patternsrS   r   r   rT   r
   r   r-  r,  get_aligned_spans_x2y)r  rl  rM  rW   patternsrulerrX   r   rm  r{   r  r   	ents_predents_x2ys                 rY   test_aligned_spans_x2yr  n  s   :D
)C'9:$:;H LL(E	x 
d)C,/HH5SSYY 5&&9IIII(F	
C"#X.	Vc&#99:EBH RJzx%PQG,3,=,=,B,BCSSYY CPVGWWWW!!&&I,56SSYY 666:JJJJ,,Y7H,45SSYY 5&&9IIII 6 D 75s   FFFFc           	         d}t               } ||      }|j                  |      }g }d}|j                  |j                  t	        |      t	        |dz         d             |j                  |j                  t	        |      t	        |dz         d             d}||j
                  |<   t        ||      }	|	j                  j
                  |   }
|
D cg c]  }|j                  |j                  f c}d	d
gk(  sJ |	j                  |
d      }|D cg c]  }|j                  |j                  f c}d	gk(  sJ |	j                  |
d      }|D cg c]  }|j                  |j                  f c}d	d
gk(  sJ y c c}w c c}w c c}w )Nr  r   rX  CITY)r   rT  VALLEYoverlap_ents)r4   r8   )r4   r5   F)allow_overlapT)r   r   r   	char_spanrT   r  r
   r-  r   r   r  )r  rl  rM  rW   rX   gold_docr  rm  	spans_keyr   
spans_goldr   spans_y2x_no_overlapspans_y2x_overlaps                 rY   test_aligned_spans_y2x_overlapr    s   +D
)C
d)C||D!HEF	LL3v;F_,D(EVT 
LLKV&<<=X 	 	

 I %HNN9c8$G""((3J,67SSYY 7FF;KKKK #88% 9  -AASSYY AfXMMM55jPT5U,=>SSYY >66BRRRR 8 B>s   E=FFc                 |     | d      }g d}t        j                  |d|i      }|j                  d      g dk(  sJ y )Nr  )NrB   rB   rE   r  rB   r  rB   r{   r   )r   r   r   r4   rD   r   r4   r   )r
   r   r)  )rl  rX   r  r   s       rY   test_gold_ner_missing_tagsr    sC    
=
>CFJj*%=>Gy)-EEEErZ   c                 Z    | d      }g d}dgt        |      z  }t        j                  |||d      }|j                  d      \  }}|j                  d      \  }}|g dk(  sJ |g dk(  sJ  | d	      }d
g}dg}t        j                  |||d      }|j                  d      \  }}||k(  sJ ||k(  sJ t	        |j
                  dgdgdgd
g      }	t	        |j
                  g dg dg dg d      }
t        |	|
      }|j                  d      \  }}|d gk(  sJ |d gk(  sJ y )NzHe pretty quickly walks away)r4   r   r4   r4   r   rC  )rP   rQ   T)projectivizeF)r4   r   r4   r4   r4   Conrailr   zDouble-Jointedr<   )rL   rh   rQ   rP   )Doubler  Jointed)TTT)amodr@   r<   )r   r   r   )rT   r
   r   get_aligned_parser   rU   )rl  rX   rP   rQ   r   
proj_headsproj_labelsnonproj_headsnonproj_labelsdoc_adoc_b	proj_depss               rY   test_projectivizer    sv   
5
6CE7SZDud%CDG%77T7JJ$+$=$=5$=$Q!M>(((O+++ y
!CCE7Dud%CDG%77T7JJ$ 		*+UG6(STRUE 		(!&E eU#G#5545HJ	$rZ   c                      g d} g d}g d}t        |       }||k(  sJ t        j                  t              5  t        |       d d d        y # 1 sw Y   y xY w)N)rB   rB   rE   rF   rB   rC   )rB   rB   rE   r  rB   rh  )rB   rB   "rE   rF   )r   r  r  r  )good_iob
good_biluobad_iobconverted_biluos       rY   test_iob_to_biluor    sR    <H>J/G"8,O(((	z	" W  s   AAc                 H   | j                   }| D cg c]  }|j                   }}| D cg c]  }|j                   }}| D cg c]  }|j                   }}| D cg c]  }t	        |j
                         }}| D cg c]  }|j                   }}| D cg c]  }|j                   }}| D cg c]  }|j                  j                   }	}| j                  }
| j                  D cg c]%  }|j                  |j                  |j                  f' }}t               5 }t!               }|dz  }t#        j$                  |t'        |       g       |dz  }t)        | g      j+                  |       t-        |      }t/         ||            }d d d        t1        |       t3        d D              k(  sJ |d   }||j4                  j                   k(  sJ ||j4                  D cg c]  }|j                   c}k(  sJ ||j4                  D cg c]  }|j                   c}k(  sJ ||j4                  D cg c]  }|j                   c}k(  sJ ||j4                  D cg c]  }t	        |j
                         c}k(  sJ ||j4                  D cg c]  }|j                   c}k(  sJ ||j4                  D cg c]  }|j                   c}k(  sJ |	|j4                  D cg c]  }|j                  j                   c}k(  sJ ||j4                  j                  D cg c]%  }|j                  |j                  |j                  f' c}k(  sJ d|j4                  j                  v sJ d|j4                  j                  v sJ |
d   |j4                  j                  d   k(  sJ |
d   |j4                  j                  d   k(  sJ y c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w # 1 sw Y   exY wc c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w )Nzroundtrip.jsonzroundtrip.spacy)r   c              3   2   K   | ]  }t        |        y wrm   )rT   ).0r   s     rY   	<genexpr>z0test_roundtrip_docs_to_docbin.<locals>.<genexpr>  s     ?r3r7?s   r   rJ   rK   )rM  idxr   pos_strmorphlemma_dep_rD  r   rV   rS   r   r   r   r   r   srsly
write_jsonr   r   r   r	   r   rT   sumr-  )rX   rM  r   r  rM   rN   rO   rR   rQ   rP   rV   erS   r   reloaded_nlp	json_filer   r   reloaded_examplesreloaded_examples                       rY   test_roundtrip_docs_to_docbinr    s   88D
Q155
C
 qAFF D 
a166
C
$'(qc!''l(F( #$1ahh$F$ qAFF D "#!QVVXX#E#88D:=((CQQ\\1::qxx0CDC	 76y--	\#%6$7800SE"";/$ !567 s8s?->?????(+#--22222"2"<"<=Q155====$4$>$>?qAFF????#3#=#=>a166>>>>,<,F,FGqc!''lGGGG(8(B(BC1ahhCCCC$4$>$>?qAFF????'7'A'AB!QVVXXBBBB6F6P6P6U6U12qzz188,    '1166666'1166666>-77<<XFFFF>-77<<XFFFFG  
($ #C7 7 >?>GC?Bsk   OOOO>OO 0O%)*O*A-O/#O<P5PPP9P"P*P/O9c                 `   d| j                   d<   t               }t               5 }|dz  }t        | gd      j	                  |       t               j                  |      j                  |j                        }t        |      d   }d d d        j                   d   dk(  sJ y # 1 sw Y   xY w)NTcheckuserdata.spacyr   store_user_datar   )		user_datar   r   r   r   	from_diskget_docsrU   r   rX   rW   r   r   reloaded_docsreloaded_docs         rY    test_docbin_user_data_serializedr    s    !CMM'
)C	 .6//SE4088E**;7@@KM*1-	. !!'*d222. .s   A#B$$B-c                 h   t               | j                  d<   t               }t               5 }|dz  }t	        | gd      j                  |       t	               j                  |      j                  |j                        }t        |      d   }d d d        dj                  vsJ y # 1 sw Y   xY w)Nr  r  Fr  r   )
setr  r   r   r   r   r  r  rU   r   r  s         rY   $test_docbin_user_data_not_serializedr    s     UCMM'
)C	 .6//SE5199+F**;7@@KM*1-	. ,00000. .s   A#B((B1ztokens_a,tokens_b,expected)r4  r5  r6  abr6  rD   )r4  r5  r  r6  zab"r   rD   r   r4   r4  bc)r  r6  r7  )r4  r5  cdr3  rx  )r4  ''',)za'r  r  c                 d    t        | |      \  }}||f|k(  sJ t        ||       \  }}||f|k(  sJ y rm   r   )tokens_atokens_brp  a2bb2as        rY   
test_alignr    sH    : h1HC:!!!h1HC:!!!rZ   c                     d} | |      }dg}dg}dg}dg}t        j                  |||||d      }|j                         }|ddgk(  sJ |j                  dd	
      d dgk(  sJ y )Nz ar4  U-DATEr<   r   rL   r{   rQ   rP   rB   DEPTr"  r
   r   r:  r)  	rl  rM  rX   rY  r{   rQ   rP   r   r<  s	            rY   test_goldparse_startswith_spacer   C  s    D
t
CJzH8DCEzxPUVG &&(HX&&&u5$GGGrZ   c                     d} | |      }dg}dg}dg}dg}t        j                  |||||d      }|j                         }|ddgk(  sJ |j                  dd	
      dd gk(  sJ y )Nza
r4  r  r<   r   r  rB   r  Tr"  r  r  s	            rY   test_goldparse_endswith_spacer  R  s    D
t
CJzH8DCEzxPUVG &&(H#&&&u5&$GGGrZ   c                      t               }  | d      }t        j                  |ddddi      }|j                  dd      g d	k(  sJ |j                  j
                  d
   sJ |j                  j
                  d   rJ y)z,Test that the Example constructor works finezThis is a sentencerV   rH   rI   )cat1cat2r   Tr"  )Thisr`   r4  sentencer  r  N)r   r
   r   r)  r-  rV   )rW   rX   r   s      rY   test_gold_constructorr  a  s    
)C
"
#CfsC.H%IJGv6 ;    !!&)))  %%f----rZ   c                  D    dddgifddddgifdddgifg} t        |        y	)
zTest tuple format#Uber blew through $1 million a weekr{   r   r6   ORGSpotify steps up Asia expansionr   r7   r  r      r   !Google rebrands its business appsr   r5   r  N)_train_tuplesr   s    rY   test_tuple_format_implicitr  p  sJ     
/m_0MN--9:	
 
-zM?.KLJ *rZ   c                      dddgifddddgifddd	gifg} t        j                  t              5  t        |        d
d
d
       y
# 1 sw Y   y
xY w)z:Test that an error is thrown for an implicit invalid fieldr
  frumbler  r  r{   r  r  r  r  N)r  r  KeyErrorr  r  s    rY   "test_tuple_format_implicit_invalidr    sk     
/]O0LM--9:	
 
-zM?.KLJ 
x	  "j!" " "s   AAc           	         t               }|j                  d      }|j                  d       |j                  d       g }| D ]<  }|j                  t	        j
                  |j                  |d         |d                > |j                         }t        d      D ]7  }i }t        |t        ddd	      
      }|D ]  }	|j                  |	||        9 y )Nry   r  r   r   rD   r8   g      @g      @@gjt?)sizer   )r   r|   r}   r   r
   r   r   r~   r   r   r   r   )
r   rW   ry   r   r   r   r   r   batchesbatchs
             rY   r  r    s    
)C
,,u
CMM%MM%N Kg//QqT0BAaDIJK I1X <NS$1NO 	<EJJu)FJ;	<<rZ   c                    t               }t        j                  t        |j                  | d   | d         |       }|j
                  dk(  sJ |j                         }t        |      dk(  sJ |d   j
                  dk(  sJ |d   j
                  d	k(  sJ |d   j                         d
   }|d   g dk(  sJ |d   g dk(  sJ |d   g dk(  sJ |d   j                         d
   }|d   g dk(  sJ |d   g dk(  sJ |d   g dk(  sJ y )NrL   rh   r   zHi there everyone It is just mer   r   zHi there everyone rD   zIt is just metoken_annotationr   )r\   r]   r^   r  )rc   rd   re   r   )rD   r   r   )r_   r`   ra   rb   )re   rf   rd   re   )rD   r   r   r   )	r   r
   r   r   rU   rM  r   rT   to_dict)rk   rW   r   rZ  token_annotation_1token_annotation_2s         rY   test_split_sentsr#    sI   
)CCII[1+h:OPG <<<<<<((*N~!###!!!%9999!!!_444'*2245GHf%)DDDDe$(????l+y888'*2245GHf%)CCCCe$(FFFFl+|;;;rZ   c                  p   g d} g d}t        j                  | |      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ y )N)r   listenedr!   obamar  spodcastsr&   r   r%  r!   r&  r   r(  r&   rD   rD   rD   rD   rD   rD   rD   rD   r   rD   r   r4   r6   r6   r8   r5   rD   rD   rD   rD   r   rD   rD   r   rD   r   r4   r6   r8   r5   r7   r   from_stringsr   x2ylengthsr   y2xother_tokensspacy_tokensaligns      rY   test_alignmentr7        NLJL""<>E		!!"&>>>>		#;;;;		!!"&;;;;		#;;;;rZ   c                  n   t        g ddgg g dddgg      } t        | j                        g dk(  sJ t        | j                        g dk(  sJ t        | d         g dk(  sJ t        | d         g k(  sJ t        | d	         g dk(  sJ t        | d
d       g dk(  sJ t        | d
d        g dk(  sJ t        | d d       g dk(  sJ t        | d d        t        | j                        k(  sJ t        | dd       g k(  sJ t        | dd       g k(  sJ t        | dd       g k(  sJ t	        j
                  t        d      5  | d dd    d d d        t	        j
                  t        d      5  | g d    d d d        t        g g dddgg      } t        | d         g k(  sJ t        | dd
       g k(  sJ t        | d         ddgk(  sJ t        | dd       g dk(  sJ t        g dddgg g      } t        | d         g k(  sJ t        | d	d        ddgk(  sJ y # 1 sw Y   xY w# 1 sw Y   xY w)Nr  r4   )r6   r8   r5   r7   rG   r   )
r   rD   r   r4   r6   r8   r5   r7   rG   r   )r4   rD   r   r6   r   r   rJ  rD   r6   )r4   r6   r8   r5   r7   )r4   r6   r8   r5   r7   rG   r   )r   rD   r   r4   r   r   z&only supports slicing with a step of 1matchz.only supports indexing using an int or a slicer   rD   r4   )rD   r   r4   r8   )r   r   r   r1  r  r  r  )r4  s    rY   test_alignment_arrayr=    sZ   	A3L1a&ABA<9999		?o---!:%%%!:";,&&&!A<?***!";////"1;,&&&!:aff%%%!A<2!A<2"R>R	z)R	S 	%1R%	K
  	
)
 	I1v./A!:!A<2!:!Q!A<9$$$	Aq62./A";""#<Aq6!!!  s   	H2H+H(+H4c                  p   g d} g d}t        j                  | |      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ y )N)r   r%  r!   r&  r  r'  r(  r&   )r   r%  r!   Obamar   PODCASTSr&   r*  r+  r,  r-  r.  r3  s      rY   test_alignment_case_insensitiverA    r8  rZ   c                  p   g d} g d}t        j                  | |      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ y )Ni listened tor&  r  r'  r(  r&   r   r%  r!   r&  r   	podcasts.r4   rD   rD   rD   rD   rD   r   rD   r   r4   r6   r6   r8   r8   rD   rD   rD   rD   r   r   r   r   r   rD   r   r4   r6   r8   r.  r3  s      rY   test_alignment_complexrK    s    HLFL""<>E		!!"&8888		#;;;;		!!"&8888		#;;;;rZ   c                    g d}g d}t        | |g d      }t        | |g d      }|j                  dk(  sJ |j                  dk(  sJ t        ||      }|j                  }t	        |j
                  j                        g dk(  sJ t	        |j
                  j                        g dk(  sJ t	        |j                  j                        g d	k(  sJ t	        |j                  j                        g d
k(  sJ y )NrC  rE  )TFFTFFr   )TTTFTFzi listened to obama's podcasts.rG  rH  rI  rJ  )	r   rM  r
   	alignmentr   r0  r1  r   r2  )r  r4  r5  r,  r-  r   r6  s          rY   test_alignment_complex_examplerN    s    HLFL-UI -SI >>>>>>>>>>>>i+GE		!!"&8888		#;;;;		!!"&8888		#;;;;rZ   c                      g d} g d}t        j                  t              5  t        j                  | |       d d d        y # 1 sw Y   y xY w)N)sher%  r!   r&  r   r(  r&   r)  )r  r  r  r   r/  )r4  r5  s     rY   test_alignment_different_textsrQ    s=    LLJL	z	" ;|\:; ; ;s   AAc                 D   g d}g d}t        j                  ||      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ g d}g d}t        j                  ||      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g dk(  sJ t        |j                  j
                        g d	k(  sJ g d}g d
}t        j                  ||      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ g d}g d}t        j                  ||      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ g d}g d}t        j                  ||      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ g d}g d}t        j                  ||      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ g d}g d}t        j                  ||      }t        |j                  j                        g dk(  sJ t        |j                  j
                        g dk(  sJ t        |j                  j                        g d k(  sJ t        |j                  j
                        g d!k(  sJ g d"}g d#}t        j                  ||      }t        |j                  j
                        g d$k(  sJ t        |j                  j
                        g d%k(  sJ d&d'g}g d(}t        j                  ||      }d'd&g}d'd)g}t        j                  ||      }y )*N)rx  rD  r&  r  r'  r(  r&   rE  )r   r4   rD   rD   rD   rD   rD   rH  rI  )rD   rD   rD   r   r4   r6   r8   r5   )rx  rx  rD  r&  r  r'  r(  r&   )r   r   r4   rD   rD   rD   rD   rD   )r   r   r   r4   r6   r8   r5   r7   )rx  r   r%  r!   r&  r   rF  )rD   r   r4   rD   rD   rD   rD   rD   )	r   rD   r   r4   r6   r8   r8   r5   r5   )rD   rD   rD   rD   rD   r   r   )	r   r   r   r   r4   r6   r8   r5   r7   )  r   r%  r!   r&  r   rF  )rD   rD   r4   rD   rD   rD   rD   rD   )
r   r   rD   r   r4   r6   r8   r8   r5   r5   )r   rD   rD   rD   rD   r   r   )
r   rD   r   r   r   r4   r6   r8   r5   r7   )rD  r&  r  r'  r(  r&   rx  )r4   rD   rD   rD   rD   rD   r   rJ  )rD  r&  r  r'  r(  r&   rx  rx  )r   r%  r!   r&  r   rF  rx  )r4   rD   rD   rD   rD   rD   rD   r   )	r   rD   r   r4   r6   r6   r8   r8   r5   )rD   rD   rD   rD   r   r   rD   )	r   r   r   rD   r   r4   r6   r8   r5   )r   r%  r!   r&  r   rF  rS  )r4   rD   rD   rD   rD   rD   rD   rD   )
r   rD   r   r4   r6   r6   r8   r8   r5   r5   )rD   rD   rD   rD   r   r   r   )
r   r   r   rD   r   r4   r6   r8   r5   r7   )r4  z 
 r5  r6  )r4  r5  rx  r6  r<  )r   r   r4   rx  r4  )rS  r4  rx  rS  r.  )r  r4  r5  r6  s       rY   test_alignment_spacesrT  
  s   MLFL""<>E		!!"&;;;;		#;;;;		!!"&8888		#;;;; SLFL""<>E		!!"&>>>>		#;;;;		!!"&8888		#;;;; SLKL""<>E		!!"&>>>>		#>>>>		!!"&;;;;		#>>>> SLLL""<>E		!!"&>>>>		#AAAA		!!"&;;;;		#AAAA NLFL""<>E		!!"&;;;;		#;;;;		!!"&8888		#;;;; SLKL""<>E		!!"&>>>>		#>>>>		!!"&;;;;		#>>>> SLLL""<>E		!!"&>>>>		#AAAA		!!"&;;;;		#AAAA +L'L""<>E		9,,,		9,,, :L#L""<>E:L;L""<>ErZ   c                 ^   | j                  dg      }t        | j                  | D cg c]  }|j                   c}      j	                  dg|      }t        | j                  | D cg c]  }|j                   c}      j	                  dg|      }t        ||      }g d}g d}|j                  dd      |k(  sJ |j                         5 }|j                  |dd	        |j                  |d
d        d d d        |j                  dd      |k(  sJ y c c}w c c}w # 1 sw Y   ,xY w)Nr  r  r   )Nr   r    r!   Nr$   r%   r&   r   Tr"  r   r   r8   r7   )	to_arrayr   rU   rM  r&  r
   r)  
retokenizemerge)	rX   r4  r   doc1doc2r   	expected1	expected2retokenizers	            rY   test_retokenized_docsr^  [  s   eWAsyy 5A 56AA5'1MDsyy 5A 56AA5'1MDdD!GbIJIv6)CCC		 %k$q)$$q)$% v6)CCC !6 5% %s   D
'D
)D##D,c                      d } fd}t        j                  d      }|j                  d       t               }t	        || |       d dddddg g |	      }t        j                  t        d
      5  |D ]  } 	 d d d        y # 1 sw Y   y xY w)Nc                 @    |d   dk(  sJ |d   dk(  sJ t        d      )Nstepr   epochrD   ran_before_update)r  )rW   argss     rY   before_updatez2test_training_before_update.<locals>.before_updatel  s4    F|q   G}!!!
 ,--rZ   c               3   0   K   dt                gf y w)NrD   )r
   )rX   s   rY   generate_batchz3test_training_before_update.<locals>.generate_batchu  s     '#s#$$$s   entaggerc                       y rm   rj   rj   rZ   rY   r   z-test_training_before_update.<locals>.<lambda>  s    rZ   g?d   r   )dropouteval_frequencyaccumulate_gradientpatience	max_stepsexcludeannotating_componentsre  rc  r:  )spacyblankr|   r   r   r  r  r  )rX   re  rg  rW   r   	generatorr   s   `      rY   test_training_before_updaterv  k  s    .% ++d
CLLI% #I 
z)<	=  	A	  s   2BB)]r   r'  r  r  	thinc.apir   r   rs  spacy.lang.enr   spacy.tokensr   r   spacy.trainingr   r	   r
   r   r   r   r   r   spacy.training.alignr   spacy.training.alignment_arrayr   spacy.training.convertersr   spacy.training.loopr   
spacy.utilr   r   r   r   utilr   fixturerX   rk   rU   markissuer   r   r   r   r  r	  r  r  r  r.  r1  r=  r@  filterwarningsrR  r[  rn  rq  rv  r{  r  r  r  r  r  r  r  r  r  r  r  r  parametrizer  r   r  r  r  r  r  r#  r7  r=  rA  rK  rN  rQ  rT  r^  rv  rj   rZ   rY   <module>r     s       '  ! $	 	 	 0 9 2 5    ! !H    
 3-* -*` 4V* V*r&R 4V V(1:C	-2	,,03 1236 336l966D ,E*C$G,	(	$J&J4S>F!H%GP
31  	4+!qcA31a&1#'GH CLcA3aS!Is#34	

 ttSkaS1a&MQFQC=#AB!fqcA31#sQF!34	
  cA3A1#sQC!!56	

 scUb1#Y!./!cAq6A3$1v1vs&;<	
'8"98"HH."< <*<"@<<<&;N?bD !rZ   