
    i>)                     f   d dl Z d dlmZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZ d dlmZmZ d dlmZmZmZ dd	lmZ edd
       Zej,                  d        Zej,                  d        Zej2                  j5                  d      d        Zd Zej2                  j5                  d      d        Zd Zy)    N)contextmanagerEnglish)contains_cycle)DocDocBinSpan)CorpusExample)create_lower_casing_augmentercreate_orth_variants_augmentermake_whitespace_variant   )make_tempdirc              #      K   t               5 }||z  }t        |       j                  |       | d d d        y # 1 sw Y   y xY ww)N)docs)r   r   to_disk)r   nametmpdiroutput_files       u/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/spacy/tests/training/test_augmenters.pymake_docbinr      sC     	 6tmD!!+.  s   A%;	AA Ac                      t               S )Nr        r   nlpr      s
    9r   c                 r    g d}g d}g d}g d}ddd}t        | j                  ||||      }||_        |S )	N)
Sarahz'ssisterflewtoSiliconValleyviaLondon.)
NNPPOSNNVBDINr'   r'   r+   r'   r&   )
PROPNPARTNOUNVERBADPr,   r,   r0   r,   PUNCT)
zB-PERSONzI-PERSONO r2   zB-LOCzI-LOCr2   B-GPEr2         ?        )TRAVELBAKING)wordstagsposents)r   vocabcats)r   r9   r:   r;   r<   r>   docs          r   r?   r?       sE     _ELD
]CVDS)D
ciiu4St
DCCHJr   zignore::UserWarningc                 "   dgddgddgg ddg}g d}g d}d	gt        |      z  }d
|d<   d
|d<   t        | j                  |||      }t        ddd|i      }t	        |gdz        5 }t        ||      }t         ||              d d d        t        ddd|i      }t	        |gdz        5 }t        ||      } ||       D ]<  }	|	j                  D ]+  }
|
j                  |
j                  j                         k(  r+J  > 	 d d d        t        | j                  |d	gt        |      z        }t        ddd|i      }t	        |gdz        5 }t        ||      } ||       D ]I  }	t        |	j                  |      D ].  \  }}|j                  |j                  j                         k(  r.J  K 	 d d d        t        | j                  |d	gt        |      z        }t        ddd|i      }t	        |gdz        5 }t        ||      } ||       D ];  }	t        |	j                  |      D ]   \  }}|j                  |j                  k(  r J  = 	 d d d        y # 1 sw Y   xY w# 1 sw Y   hxY w# 1 sw Y   xY w# 1 sw Y   y xY w)NNFP   …...)r:   variants:)-   —   –-----   ——)z

A	BabrB   rC   rF   rG   rH   rI   rJ   rK   )_SPr)   rM   r)   r)   r)   rA   rA   rE   rE   rE   rE   rE   rE   TFr   r   )r9   spacesr:   g?      ?single)levellowerorth_variants
   	augmenterr5   )r9   rR   r6   )lenr   r=   r   r   r
   list	referencetextrV   zip)r   rT   r9   r:   rR   r?   rZ   r   readerexampletokenex_token	doc_tokens                r   test_make_orth_variantsre   .   s    uen5$NOF
 gE\DVc%j FF1IF1I
ciiuV$
?C.Xv,>I 
cURZ	  Ky9VC[ /Xv,>I 
cURZ	  8Ky9c{ 	8G ** 8zzUZZ%5%5%77778	88 ciiudVc%j-@
AC.Xv,>I 
cURZ	  ?Ky9c{ 	?G'*7+<+<c'B ?#)}}	(<(<(>>>>?	?? ciiudVc%j-@
AC.Xv,>I 
cURZ	  7Ky9c{ 	7G'*7+<+<c'B 7#)}}	6667	77 7? 8 8? ?7 7sE   *I0AI, I,AI9/I9AJJI),I69JJc                 ~   t        d      }t        |g      5 }t        ||      }t         ||             }d d d        d   }|j                  j
                  |j
                  j                         k(  sJ |j                  j
                  |j
                  j                         k(  sJ |j                  D cg c]%  }|j                  |j                  |j                  f' }}|j                  j                  D cg c]%  }|j                  |j                  |j                  f' c}|k(  sJ t        |j                  j                  |j                        D ].  \  }	}
|	j
                  |
j
                  j                         k(  r.J  |D cg c]  }|j                   c}|j                  D cg c]  }|j                   c}k(  sJ |j                  D cg c]  }|j                   c}|D cg c]  }|j                   c}k(  sJ g d}t        | j                   |      }t        |g      5 }t        ||      }t         ||             }d d d        |d   }|j                  j
                  |j
                  j                         k(  sJ |j                  j
                  |j
                  j                         k(  sJ |j                  D cg c]  }|j
                   c}|D cg c]  }|j                          c}k(  sJ |j                  D cg c]  }|j
                   c}| j#                  |j
                  j                               D cg c]  }|j
                   c}k(  sJ y # 1 sw Y   =xY wc c}w c c}w c c}w c c}w c c}w c c}w # 1 sw Y   NxY wc c}w c c}w c c}w c c}w )Nr5   )rU   rY   r   )rL   rN   zCCC.)r9   )r   r   r
   r\   r]   r^   rV   	predictedr<   startendlabelr_   ent_iobpos_r   r=   make_doc)r   r?   rZ   r   r`   corpuseger<   ref_entorig_enttr9   s                r   test_lowercase_augmenterrt   e   s   -C8I	cU	 #{y9fSk"# 
B<< 0000<< 0000/2xx8!QWWaeeQWW%8D8/1||/@/@A!QWWaeeQWW%ATIII !2!2CHH= 5||x}}2244445"#!AII#2<<'Ha		'HHHHLL)qAFF)c-Baff-BBBB E
ciiu
%C	cU	 #{y9fSk"# 
B<< 0000<< 0000LL)qAFF)-GAaggi-GGGGLL)qAFF)SXX^^%56..   1# # 9A $'H)-B# # *-G) .sS   M39*N =*NN
1NN2N3NN+,N0N5N:3M=N(c                    ddt         fd}t        |g      5 }t        | |             }t         ||             }d d d        d}d}d   j                  |k(  sJ |d   j
                  j                  |k(  sJ |d   j                  j                  |k(  sJ |d   j                  |k(  sJ |d   j
                  j                  |k(  sJ |d   j                  j                  |k(  sJ |j                  D cg c]%  }|j                  |j                  |j                  f' }	}|d   j
                  j                  D cg c]%  }|j                  |j                  |j                  f' c}|	k(  sJ |d   j
                  j                  D cg c]%  }|j                  |j                  |j                  f' c}|	k(  sJ y # 1 sw Y   xY wc c}w c c}w c c}w )	N	randomizec                       fd}|S )Nc              3   $  K   |j                   }	rE|D cg c]9  }t        j                         dk  r|j                         n|j                         ; }}n@t	        |      D cg c]*  \  }}|dz  r|j                         n|j                         , }}}|j                         }| j                  dj                  |            }|D cg c]  }|j                    c}|d   d<   | |j                  ||       y c c}w c c}}w c c}w w)NrS   r   r3   token_annotationORTH)	r^   randomrV   upper	enumerateto_dictrm   join	from_dict)
r   ra   r^   cchiexample_dictr?   rs   rv   s
            r   augmentzRtest_custom_data_augmentation.<locals>.create_spongebob_augmenter.<locals>.augment   s     <<DQUVA6==?S#8aggiaggiGVVDMdOTDAq1q5aggiaggi7TT"??,L,,rwwr{+CHK7L17LL+,V4M##C66 WT 8Ms(   D>D D%/D7DD2Dr   )rv   r   s   ` r   create_spongebob_augmenterzAtest_custom_data_augmentation.<locals>.create_spongebob_augmenter   s    
	7 r   rY   z4Sarah 's sister flew to Silicon Valley via London . z4SaRaH 's sIsTeR FlEw tO SiLiCoN VaLlEy vIa lOnDoN . r      )F)boolr   r
   r\   r^   r]   rg   r<   rh   ri   rj   )
r   r?   r   r   r`   rn   	orig_text	augmentedrp   r<   s
             r   test_custom_data_augmentationr      s   d  
cU	 #{/I/KLfSk"# GIFI!9>>Y&&&!9##y000!9##y000!9>>Y&&&!9##y000!9##y000/2xx8!QWWaeeQWW%8D8/5ay/B/B/G/GH!QWWaeeQWW%HDPPP/5ay/B/B/G/GH!QWWaeeQWW%HDPPP# # 9HHs   $F=6*G
=*G
*G=Gc           
      
   d}g d}g d}g d}g d}g d}g d}g d}t        | j                  |||||||	      }	|	j                  |k(  sJ t        | j	                  |      |	      }
t        | |
d
d      }|j                  j                  d   j                  dk(  sJ t        | |
d
d      }|j                  j                  d   j                  dk(  sJ t        | |
d
d      }|j                  j                  d   j                  dk(  sJ t        | |
d
d      }|j                  j                  d   j                  dk(  sJ t        t        |	      dz         D ]  }t        | |
d
|      }|j                  |   j                  sJ |j                  D cg c]  }|j                   c}|d | dgz   ||d  z   k(  sJ |j                  D cg c]  }|j                   c}|d | d
gz   ||d  z   k(  sJ |j                  D cg c]  }|j                   c}|d | dgz   ||d  z   k(  sJ |j                  j                  d      rJ |j                  j                  d      rJ t        |j                  D cg c]  }|j                   j"                   c}      rJ t        t%        |	j&                              dk(  sJ |dk(  r(|j                  |   j                   j"                  dk(  s-J |j                  |   j                   j"                  |dz
  k(  sJ dD ]  }t        | |d|      }t        |j                  D cg c]  }|j                   j"                   c}      rJ t        t%        |	j&                              dk(  sJ |j                  |   j                   j"                  |dz
  k(  rJ  t        |	j                        t        |j                  j                        k(  sJ t)        d |j                  D              sJ |j                  j                  D ]$  }|d   j                  rJ |d   j                  s$J   d|
j                  d   _        t        | |
d
d      }|j                  |
j                  j                  k(  sJ d|
j                  d   _        |
j                  dd g|
j                  j*                  d<   t        | |
d
d      }|j                  |
j                  j                  k(  sJ |
j                  j*                  d= t-        |	ddd d!"      g|
j                  _        t        | |
d
d      }|j                  |
j                  j                  k(  sJ y c c}w c c}w c c}w c c}w c c}w )#Nz?They flew to New York City.
Then they drove to Washington, D.C.)Theyr    r!   NewYorkCityr&   
Thentheydrover!   
Washington,D.C.)TTTTTFFFTTTTFTF)PRPr*   r+   r'   r'   r'   r&   rQ   RBr   r*   r+   r'   r   r'   )r   flyr!   r   r   r   r&   r   thenr   driver!   r   r   r   )r   r   r         r   r   rX   rX   rX   rX   rX         r   )nsubjROOTprepcompoundr   pobjpunctdepadvmodr   r   r   r   r   appos)r2   r3   r2   r4   I-GPEr   r2   r2   r2   r2   r2   r2   r4   r2   r4   )r9   rR   r:   lemmasheadsdepsr<       r   zNew York Cityr   zNew  York Cityr   zNew York  City   r   rQ   r   r(   MORPHr   )r      rX   z		
c              3   :   K   | ]  }|j                   d k(    yw)r   N)rk   ).0rs   s     r   	<genexpr>z/test_make_whitespace_variant.<locals>.<genexpr>   s     <a199><s   r3   r   spansENTQ123)rj   kb_id)r   r=   r^   r   rm   r   r]   r<   ranger[   is_spacetag_lemma_dep_has_annotationr   headr   r\   sentsanyr   r	   )r   r^   r9   rR   r:   r   r   r   r<   r?   ra   mod_exr   rs   jmod_ex2ents                    r   test_make_whitespace_variantr      sI   MD~ElFmD~FAE RDiD
			C 88tcll4(#.G$S'3:F  #((O;;;$S'3:F  #((,<<<<$S'3:F  #((,<<<<$S'3:F  #((O;;;3s8a<  ((gsA>"++++ & 0 0111T"1X5G$qr(5RRRR"("2"23Q3vbqzSE7IFSTSUJ7VVVV & 0 0111T"1X5G$qr(5RRRR##225999##227;;;!V5E5E"F16688"FGGG4		?#q(((6##A&++--222##A&++--Q666 	8A-c68QGG%9J9J&KAqvvxx&KLLLtCII'1,,,$$Q',,..!a%777		8 388}F$4$4$9$9 ::::<6+;+;<<<<##(( 	(C1v&&2w''''	(7(B !#Ga$S'3:F;;'++00000 'Ga )0(9(9!A(>'?GG$$S'3:F;;'++00000( #31EHIG$S'3:F;;'++00000W 231
 #G 'Ls   U%U*>U/3U4
:U9)zroundtrip.spacy)r{   
contextlibr   pytestspacy.lang.enr   (spacy.pipeline._parser_internals.nonprojr   spacy.tokensr   r   r	   spacy.trainingr
   r   spacy.training.augmentr   r   r   utilr   r   fixturer   r?   markfilterwarningsre   rt   r   r   r   r   r   <module>r      s     %  ! C * * *        
 
 1237 337l> 12Q 3Q@P1r   