
    i*              
       :   d dl Z d dlZd dlmZmZ d dlmZ ej                  j                  d      d        Z	d Z
d Zd Zd	 Zd
 Zd Zd Zd Zej                  j%                  dddii gddii gddii gddiddigdddddigddig      d        Zd Zd Zd Zy)    N)DocToken)Vocabi  c           	      &   g d}t        j                  ddgddgddgdd	gd
dgddggd      }t        | |      }||_        g d}|D cg c]  }|j                   c}|k(  sJ g d}t        |      D ]  \  }}|||   _         |D cg c]  }|j                   c}|k(  sJ |D cg c]  }|j                   }	}t        |	      t        |      k(  sJ |j                         5 }
|d   df|d   g}ddgddgddgd}|
j                  |d   ddg||       d d d        g d}|D cg c]  }|j                   c}|k(  sJ g d}|D cg c]  }|j                   c}|k(  sJ |D cg c]  }|j                   }}t        |      t        |      k(  sJ |	d   j                         |d   j                         k(  sJ |	d   j                         |d   j                         k(  sJ |	d   j                         |d   j                         k(  sJ |	d   j                         |d   j                         k(  sJ |	d   j                         |d   j                         k(  sJ y c c}w c c}w c c}w # 1 sw Y   _xY wc c}w c c}w c c}w ) N)IliveinNewYorkrightnowg      ?g?g       @g @g      @g@g      @gffffff@g      @gffffff@g      @gffffff@f)dtypewords         PROPNNewYorkpobjcompound)POSLEMMADEP)headsattrs)r   r   r	   r   r   r   r   r            )numpyasarrayr   tensortext	enumeratelemma_vectorlen
retokenizesplittolist)en_vocabr   r#   doc	gold_texttoken
gold_lemmailemma	vectors_1retokenizerr   r   	vectors_2s                 v/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/spacy/tests/doc/test_retokenize_split.pytest_issue3540r7      s   :E]]
sc3Z#sc3Z#sc3ZPF he
$CCJ>I$'(5EJJ(I555?Jj) 5A&)*UELL*j888+./%/I/y>SX%%%		 M[a&!c!f%W%V_J'

 	#a&5&/eLM CI$'(5EJJ(I555CJ&)*UELL*j888+./%/I/y>SX%%%Q< IaL$7$7$9999Q< IaL$7$7$9999Q< IaL$7$7$9999Q< IaL$7$7$9999Q< IaL$7$7$99997 ) +/M M )*/s0   
I(
I-)I2'3I7*JJ	,J7Jc                 z   g d}g d}dgt        |      z  }t        | |||      }t        |      dk(  sJ t        t        |            dk(  sJ |d   j                  j                  dk(  sJ |d	   j                  j                  d
k(  sJ |j                         5 }|j                  |d   ddg|d   d	f|d	   gdgdz  ddgdgdz  dgdz  d       d d d        t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  j                  dk(  sJ |d   j                  dk(  sJ t        |d   j                        dk(  sJ |d	   j                  dk(  sJ |d	   j                  dk(  sJ |d	   j                  j                  dk(  sJ t        |d	   j                        dk(  sJ |d   j                  dk(  sJ |d   j                  j                  d
k(  sJ |d   j                  d
k(  sJ |d   j                  j                  d
k(  sJ t        t        |            dk(  sJ y # 1 sw Y   ^xY w)N
LosAngelesstart.r   r   r   depr   r   depsr      r   r;   r   r<   LosAngelesNNPr   GPEzNumber=Sing)tagr2   ent_typemorphr   r   )	r(   r   strheadr$   r)   r*   idxrH   )r,   r   r   r@   r-   r4   s         r6   test_doc_retokenize_splitrM   0   sX   (EE7SZD
he5t
<Cs8q==s3x=Bq6;;w&&&q6;;s"""		 
[FI!fa[#a&!w{+"GaK'1,		 	 
	

 s8q==q6;;%q6;;y(((q6::??s1v||---q6::??q6;;)###q6;;w&&&s1v||---q6;;'!!!q6;;s"""q6;;#q6;;s"""s3x=B3
 
s   8H00H:c                 t   g d}g d}dgt        |      z  }t        | |||      }|j                         5 }|j                  |d   ddg|d   df|d   g       d d d        |d   j                  d	k(  sJ |d   j                  d	k(  sJ g d}g d}dgt        |      z  }t        | |||      }|D ]	  }d
|_         |j                         5 }|j                  |d   ddg|d   df|d   g       d d d        |d   j                  dk(  sJ |d   j                  dk(  sJ y # 1 sw Y   xY w# 1 sw Y   >xY w)Nr9   r=   r>   r?   r   rB   rC   r    a)r(   r   r)   r*   r&   )r,   r   r   r@   r-   r4   ts          r6    test_doc_retokenize_split_lemmasrR   U   sn   (EE7SZD
he5t
<C		 
[FI!fa[#a&!	

 q6==Bq6==B )EE7SZD
he5t
<C 		 
[FI!fa[#a&!	

 q6==E!!!q6==I%%%/
 
 
 
s   #D"#D."D+.D7c           	         t        | g d      }|j                  j                  j                  d      }|j                  j                  j                  d      }|j	                         5 }|j                  |d   ddg|d   df|d   gd	||gi
       d d d        |d   j                  |k(  sJ |d   j                  |k(  sJ y # 1 sw Y   2xY w)Nr9   r   amodsubjectr   rB   rC   r   r>   rI   )r   vocabstringsaddr)   r*   r>   )r,   r-   dep1dep2r4   s        r6   &test_doc_retokenize_split_dependenciesr[   u   s    
h:
;C99  (D99  +D		 
[FI!fa[#a&!4,'	 	 	

 q6::q6::
 
s   *)CCc           
         t        | g d      }t        j                  t              5  |j	                         5 }|j                  |d   ddg|d   g       d d d        d d d        t        j                  t              5  |j	                         5 }|j                  |d   ddg|d   |d   |d   g       d d d        d d d        y # 1 sw Y   qxY w# 1 sw Y   uxY w# 1 sw Y   *xY w# 1 sw Y   y xY w)Nr9   r   r   rB   rC   r   r   pytestraises
ValueErrorr)   r*   r,   r-   r4   s      r6   %test_doc_retokenize_split_heads_errorrb      s    
h:
;C	z	" D^^ 	Dc!fui&83q6(C	DD
 
z	" T^^ 	Tc!fui&83q63q63q6:RS	TT T		D 	DD D	T 	TT TsF   CCC C+%C6C+C	CCC(	$C++C4c            	         g d} t        t               |       }|j                  j                  j	                  d      ddfg|_        |d   j                  dk(  sJ |d   j                  dk(  sJ |j                         5 }|j                  |d   g d	|d   df|d   df|d   g       d d d        |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d
   j                  dk(  sJ y # 1 sw Y   ZxY w)N)abcder   zent-abcdr   r   Br   r   )rP   bcr   )	r   r   rV   rW   rX   entsent_iob_r)   r*   )r   r-   r4   s      r6   *test_doc_retokenize_spans_entity_split_iobrl      s   E
egU
#C""&&z2Aq9:CHq6??c!!!q6??c!!!		 W[#a&/SVQK#a&!cRSf3UVWq6??c!!!q6??c!!!q6??c!!!q6??c!!!W Ws    )DDc           	         g d}g d}g d}t        | |||      }t        |j                        \  }}t        |      }t        |      }|j	                         5 }	|	j                  |d   ddg|d   df|d   gd	d
dgi       |	j                  |d   ddg|d   df|d   gd	d
dgi       d d d        t        |j                        \  }}t        |      |dz   k(  sJ t        |      |dz   k(  sJ y # 1 sw Y   HxY w)N)
StewartLeeisrP   standupcomedianr<   Helivesr	   EnglandandlovesJoePasqualer<   )r   r   r   r   r   r   r      ry   ry   	   ry   ry         )nsubjROOTdetrT   prtattrpunctr}   r~   prepr   ccconjr   r   r?   r   StewartLeer   r>   r   r}   rI      JoePasqualer|   dobj)r   listsentsr(   r)   r*   )
r,   r   r   r@   r-   sent1sent2init_len	init_len2r4   s
             r6   5test_doc_retokenize_spans_sentence_update_after_splitr      s4   KE;EGD he5t
<C		?LE55zHE
I		 
[F!fa[#a&!:w/0	 	 	
 	GJ"gq\3r7#:v./	 	 	

 		?LE5u:A%%%u:Q&&&
 
s   AC22C;c           	         t        | g d      }t        j                  t              5  |j	                         5 }|j                  |d   ddg|d   df|d   dfg       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)al  Test that the regular retokenizer.split raises an error if the orths
    don't match the original token text. There might still be a method that
    allows this, but for the default use cases, merging and splitting should
    always conform with spaCy's non-destructive tokenization policy. Otherwise,
    it can lead to very confusing and unexpected results.
    r9   r   r   LANr]   ra   s      r6   (test_doc_retokenize_split_orths_mismatchr      s     h:
;C	z	" N^^ 	Nc!fsCjCFA;Q2LM	NN N	N 	NN Ns"   A<%A0A<0A9	5A<<Bc                 t   t        j                  ddd       t        j                  ddd       t        | ddg	      }|j                         5 }|d
   df|d   g}dddddig}ddg|d}|j	                  |d
   ddg||       d d d        |d
   j
                  dk(  sJ |d
   j                  j                  du sJ |d
   j                  j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  j                  du sJ |d   j                  j                  dk(  sJ y # 1 sw Y   xY w)NrP   FTdefaultforcerh   nothingr:   r;   r   r   r   1)rP   rh   2losangeles)r2   _rB   rC   rI   )	r   set_extensionr   r)   r*   r&   r   rP   rh   )r,   r-   r4   r   
underscorer   s         r6   )test_doc_retokenize_split_extension_attrsr      sJ   	U$7	Yd;
h|W5
6C		 J[a&!c!f% s+c3Z8
 ),:>#a&5)"4e5I	J
 q6==E!!!q688::q688::q6==I%%%q688::q688::J Js   7D..D7underscore_attrsrP   xrh   ri   )rP   r   c                    t        j                  ddd       t        j                  dd d       t        j                  dd	 d
       t        | ddg      }d|i}t        j                  t
              5  |j                         5 }|d   df|d   g}|j                  |d   ddg||       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   FTr   rP   c                     | S N r   s    r6   <lambda>zCtest_doc_retokenize_split_extension_attrs_invalid.<locals>.<lambda>       a     )getterr   rh   c                     | S r   r   r   s    r6   r   zCtest_doc_retokenize_split_extension_attrs_invalid.<locals>.<lambda>   r   r   )methodr   r:   r;   r   r   r   r   rB   rC   rI   )r   r   r   r^   r_   r`   r)   r*   )r,   r   r-   r   r4   r   s         r6   1test_doc_retokenize_split_extension_attrs_invalidr      s     
U$7	Kt<	Kt<
h|W5
6C"#E	z	" N^^ 	N!fa[#a&)Ec!fui&8%uM	NN N	N 	NN Ns$   7C'C /C C		CCc                    t        | dg      d   j                  rJ t        | dg      d   j                  rJ t        | ddg      }|d   j                  rJ |j                         5 }ddd	gi}|d   d
f|d
   g}|j                  |d   ddg||       ddd       |d   j                  sJ |d
   j                  rJ y# 1 sw Y   ,xY w)a  Test that retokenization also sets attributes on the lexeme if they're
    lexical attributes. For example, if a user sets IS_STOP, it should mean that
    "all tokens with that lexeme" are marked as a stop word, so the ambiguity
    here is acceptable. Also see #2390.
    rB   r   r   rC   r:   r;   is_stopTFr   rI   N)r   r   r)   r*   )r,   r-   r4   r   r   s        r6   $test_doc_retokenizer_split_lex_attrsr      s     8E7+A.66668I;/2::::
h|W5
6C1v~~		 J[T5M*a&!c!f%#a&5)"4e5IJ q6>>>1v~~~J Js   +-CCc                 6   d}t        | |j                         dd       }|j                         5 }|d   }|dfgt        |      z  }|j                  ||j                     t        |j                        |       ddd       t        | |j                               }|j                         5 }|d   }|dfgt        |      z  }|j                  ||j                     t        |j                        |       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zB#4604: realloc correctly when new tokens outnumber original tokenszOHyperglycemic adverse events following antipsychotic drug administration in theNr   r   r   )r   r*   r)   r(   r1   r   r$   )r,   r$   r-   r4   r/   r   s         r6   test_doc_retokenizer_reallocr     s
   \D
hdjjl3B/
0C		 G[As5z)#egg,UZZ(8FG hdjjl
+C		 G[As5z)#egg,UZZ(8FG GG G
G Gs   AD/ADDDc           
         d}t        | |j                               }d|d   _        |d   }|j                         5 }|j                  |g dt	        d      D cg c]  }||f c}       ddd       |d	   j
                  d
k(  sJ |d	   j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  dk(  sJ yc c}w # 1 sw Y   _xY w)z#6060: reset norm in splitz6The quick brownfoxjumpsoverthe lazy dog w/ white spotsr   withr   r   )brownfoxjumpsoverther   Nrz   zw/r   )r   r*   norm_r)   ranger$   )r,   r$   r-   r/   r4   rL   s         r6   test_doc_retokenizer_split_normr     s    CD
hdjjl
+C CFL FE		 
[4+084CE3<4 	 	

 q6;;$q6<<6!!!q6;;&   q6<<6!!! 5	
 
s   CC%CCC)r!   r^   spacy.tokensr   r   spacy.vocabr   markissuer7   rM   rR   r[   rb   rl   r   r   r   parametrizer   r   r   r   r   r   r6   <module>r      s      #  4$: $:N"J&@
T"'<
N" 
sR
sR
sR
sc3Z 	Sz*	c

	N
	N$G"r   