
    i[                        d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d dl	m
Z
mZmZmZmZmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZmZ d d
l m!Z! ddl"m#Z# d Z$ejJ                  jM                  d      d        Z'ejJ                  jM                  d      d        Z(ejJ                  jM                  d      d        Z)ejJ                  jM                  d      d        Z*ejJ                  jW                  dg d      ejJ                  jW                  deeg      ejJ                  jM                  d      d                      Z,ejJ                  jW                  dg d      ejJ                  jM                  d      d               Z-ejJ                  jM                  d      d         Z.ejJ                  jM                  d      d!        Z/ ej`                  d"       G d# d$             Z1ejJ                  jM                  d%      d&        Z2ejJ                  jM                  d'      d(        Z3ejJ                  jW                  dg d)g      d*        Z4d+ Z5ejJ                  jW                  dd,d-g      d.        Z6d/ Z7d0 Z8d1 Z9d2 Z:d3 Z;d4 Z<ejJ                  jW                  d5g d6g d7 ejz                  g d8g d9g d:g d;g      fg d<g d= ejz                  g d>g d?g d@g dAg dBg dCg dDg dEg dFg	      fg      dG        Z>dH Z?dI Z@dJ ZAejJ                  j                  dK      dL        ZCdM ZDdN ZEdO ZFdP ZGdQ ZHdR ZIdS ZJdT ZKdU ZLdV ZMdW ZNdX ZOdY ZPdZ ZQd[ ZRy)\    N)assert_array_equal)NumpyOpsget_current_ops)	DEPENT_IOBENT_TYPEHEADIS_ALPHAMORPHPOS
SENT_STARTTAG)English)MultiLanguage)Language)Lexeme)DocSpan	SpanGroupToken)Vocab   )clean_underscorec                 n   g d}g d}t        | |g d      }|D cg c]  }|j                   c}g dk(  sJ t        | ||dgdz        }|D cg c]  }|j                   c}g dk(  sJ t        | |dgdz  |dgdz  	      }|D cg c]  }|j                   c}g dk(  sJ y c c}w c c}w c c}w )
N)abcd)r   r      r   )TFTF)wordssent_startsdep   r    headsdepsT)r    r!   r%   r&   )r   is_sent_start)en_vocabr    r%   docts        m/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/spacy/tests/doc/test_doc_api.pytest_doc_api_initr,      s     EE
he1K
LC%()AOO)-GGGG he5w{
CC%()AOO)-GGGG
D6A:U%STC &))AOO)-GGGG * *
 *s   B(B-
B2i  c                  V   g d} t        t               |       }t        |dd|j                  j                  d         g|_        |j                         5 }|j                  |dd        d	d	d	       |j
                  D cg c]  }|j                   c}sJ y	# 1 sw Y   0xY wc c}w )
z9Test that entity labels still match after merging tokens.)

worda.r.   wordb-	Biosphere2r2   z 
r          PRODUCTlabel      N)	r   r   r   vocabstringsents
retokenizemergetext)r    r)   retokenizerents       r+   test_issue1547rE   /   s     SE
egU
#CS!Qcii&7&7	&BCDCH		 $[#a(#$ #)CHH)))$ $)s   BB&B#i  c                      t        t               g d      } | d   dk  rJ | d   J | d   dk\  sJ | dd dk  rJ | dd J | dd dk\  sJ | j                  d   J | j                  d   dk  rJ y)z4Test comparison against None doesn't cause segfault.r   r   r   r5   r   Nr   r   )r   r   r=   )r)   s    r+   test_issue1757rH   :   s     eg_
-C1v}q6q6T>>2Aw~r7r7d??99S>%%%yy~$$$$    i\	  c           	      T   g d}g d}dgt        |      z  }t        j                  g dg dg dg dg dg d	gt        j                  
      }t	        | |||      }|d d  }|j                         |k(  j                         sJ |j                         |k(  j                         sJ y )N)Shecreatedr   testforspacy)r   r      r   rP   r#   r"   )r   r   r   r   r   r   )r   r   r   r   r   r   )r   r   r   rP   rP   rP   )r   r   rP   rP   rP   rP   )r   r   rP   rP   r#   r#   )r   r   rP   rP   r#   r;   dtyper$   )lennumpyarrayint32r   get_lca_matrixall)r(   r    r%   r&   matrixr)   spans          r+   test_issue2396r[   H   s    ;EE7SZD[[	
 kk
F he5t
<Cq6D F*//111!V+00222rI   i,  c                 ^    ddg}ddg}ddg}t        | |||       |ddgk(  sJ |ddgk(  sJ y )NAsentencezB-TYPE1 TF)r=   r    r?   r!   r   )r(   r    r?   r!   s       r+   test_init_args_unmodifiedra   ^   sX    *Er?D-K	 Ir?"""4-'''rI   rB   )z-0.23z+123,456u   ±1lang_clsi
  c                 d     |       } ||       }t        |      dk(  sJ |d   j                  sJ y)z2Check that like_num handles + and - before number.r   r   N)rS   like_num)rB   rb   nlpr)   s       r+   test_issue2782rf   m   s4    
 *C
d)Cs8q==q6???rI   r^   )zThe story was to the effect that a young American student recently called on Professor Christlieb with a letter of introduction.zThe next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale's #1.zThe next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale's number onezIndeed, making the one who remains do all the work has installed him into a position of such insolent tyranny, it will take a month at least to reduce him to his proper proportions.zLIt was a missed assignment, but it shouldn't have resulted in a turnover ...i  c                     t               } ||       }d}|D ]  }||j                  z  } ||j                  t              j	                  dd      k(  sJ y)z8Test that the Doc's count_by function works consistentlyr   r   N)r   is_alphacount_byr
   get)r^   re   r)   counttokens        r+   test_issue3869rm   x   sZ     )C
h-CE   CLL*..q!4444rI   iz  c                 &   g d}g d}g d}t        | |||      }|dd }|j                         }|j                         }|sJ |d   j                  j                  dk(  sJ |d   j
                  d	k(  sJ |d   j                  j                  dk(  sJ |d   j
                  d
k(  sJ |d   j                  j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  j                  dk(  sJ |d   j
                  d	k(  sJ t        t        |j                              dk(  sJ |dd }|j                         }	|	j                         }
|
sJ |	d   j                  j                  dk(  sJ |	d   j
                  dk(  sJ |	d   j                  j                  dk(  sJ |	d   j
                  dk(  sJ |	d   j                  j                  dk(  sJ |	d   j
                  d	k(  sJ t        t        |	j                              dk(  sJ y)Ensure that as_doc does not result in out-of-bound access of tokens.
    This is achieved by setting the head to itself if it would lie out of the span otherwise.
    )Hejestsatscars,thatneverfeltr   woundr0   )r   r<   r   r   r<   r<   r<   r<   	   r<   r<   )nsubjccomppreppobjpunctrz   negROOTdetdobjr~   r$   r   r;   r   rq   r"   r|   r   rr   r}   rP   r6   ry   rw   r   r   N)	r   as_docto_jsonheadrB   dep_rS   listsents)r(   r    r%   r&   r)   span2doc2	doc2_jsonspan3doc3	doc3_jsons              r+   test_issue3962r      s    \E-EfD
he5t
<C!HE<<>DI97<<'''7<<5   7<<'''7<<6!!!7<<$$$7<<6!!!7<<'''7<<5   tDJJ A%%%!HE<<>DI97<<&&&7<<5   7<<&&&7<<6!!!7<<&&&7<<5   tDJJ A%%%rI   c                 ~   g d}g d}g d}t        | |||      }|dd }|j                         }|j                         }|sJ |d   j                  j                  dk(  sJ |d   j
                  d	k(  sJ |d   j                  j                  dk(  sJ |d   j
                  d
k(  sJ |d   j                  j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  j                  dk(  sJ |d   j
                  dk(  sJ t        |j                        }t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ y)ro   )rp   rq   rr   rs   r0   Theyrv   rw   r   rx   r0   )r   r   r   r   r   r<   r<   r<   ry   r<   r<   )rz   r   r|   r}   r~   rz   r   r   r   r   r~   r$   r   r<   r   rq   r   r|   r   rr   r}   rP   r~   r#   r   r"   zjests at scars .z
They neverN)	r   r   r   r   rB   r   r   r   rS   )	r(   r    r%   r&   two_sent_docr   r   r   r   s	            r+   test_issue3962_longr      s    \E-EeDxuEEL1E<<>DI97<<'''7<<6!!!7<<'''7<<6!!!7<<$$$7<<6!!!7<<'''7<<7"""7<<&&&7<<5   7<<&&&7<<5   Eu:??8==....8==L(((rI   my_pipec                   *    e Zd ZddZd Zed        Zy)
CustomPipec                     || _         t        j                  d| j                         t	        j                  dd        y )Nmy_ext)getterdefault)namer   set_extension_get_my_extr   )selfre   r   s      r+   __init__zCustomPipe.__init__   s/    	8D,<,<=(D1rI   c                     g }|j                   D ]@  }| j                  |      }|j                  j                  d|       |j	                  |       B |j                  j                  ddj                  |             |S )Nr   r.   )r   r   _setappendjoin)r   r)   gathered_extsentsent_exts        r+   __call__zCustomPipe.__call__   sl    II 	*D''-HFFJJx*)	*
 			(DIIl34
rI   c                 ,    t        | j                        S N)strendrZ   s    r+   r   zCustomPipe._get_my_ext   s    488}rI   N)r   )__name__
__module____qualname__r   r   staticmethodr    rI   r+   r   r      s     2
  rI   r   i'  c                  J   t               } | j                  d       | j                  dd       g d}t        t               t              rYt        | j                  |d            }|d   j                  dk(  sJ |d	   j                  d
k(  sJ |d   j                  dk(  sJ yy)zQEnsure that this runs correctly and doesn't hang or crash on Windows /
    macOS.sentencizerr   )after)I like bananas.Do you like them?No, I prefer wasabi.r   )	n_processr   r   r   r   r   N)r   add_pipe
isinstancer   r   r   piperB   )re   rB   docss      r+   test_issue4903r      s     )CLLLL-L0KD/#X.CHHTQH/0Aw||0000Aw||2222Aw||5555	 /rI   i  c           	          g d}g d}g d}g d}g d}| j                   }|D ]  }|j                  |        |D cg c]  }|j                  |       }	}|D 
cg c]  }
|j                  |
       }}
|D cg c]  }|j                  |       }}t        t        t        g}t        j                  t        t        ||	|            d      }t        | ||      }|j                  ||       |D cg c]%  }|j                  |j                  |j                  f' }}t        | ||||	      }|D cg c]%  }|j                  |j                  |j                  f' }}||k(  sJ y c c}w c c}
w c c}w c c}w c c}w )
N)Thisisr   r^   )DETVERBr   NOUN) r   r   r_   )r"   adjnnatm)DTVBZr   NNuint64rQ   )r    spaces)r    posr&   tags)r>   addr   r   r   rT   rU   r   zipr   
from_arrayrB   pos_tag_)r(   r    pos_sr   deps_stags_sr>   wr   r&   pr   r*   r   attrsrU   r)   rl   v1r   v2s                        r+   test_issue5048r      sS   +E*E F(F&FG A$*+qGKKN+D+#(
)a7;;q>
)C
)$*+qGKKN+D+#sOEKKSdD12(CE
heF
3CNN5% <?	@55::uzz5::
.	@B	@xu%f6JD<@	A55::uzz5::
.	AB	A8O8 ,
)+
 
A	As   E'E,9E13*E63*E;)onetwothreec                     t        | |      }|d   }|d   }|d   }|\  }}}||cxk  r|k  sJ  J ||kD  rJ ||kD  sJ ||k  sJ ||k\  sJ y )Nr5   r`   )r(   rB   r)   token3token2token1s         r+   'test_doc_api_compare_by_string_positionr     s    
hd
#CWFWFWF FFFF#V#####F??VVrI   c                    d} | |      }|d   j                   dk(  sJ |d   j                   dk(  sJ t        j                  t              5  |t	        |          d d d        d }|dd } ||      rJ |dd } ||      d	k(  sJ |ddd   } ||      d	k(  sJ t        j                  t
              5  |ddd
    d d d        t        j                  t
              5  |ddd    d d d        |dd } ||      dk(  sJ |dd } ||      dk(  sJ |dd } ||      dk(  sJ |dd }|j                  |j                  cxk(  rdk(  rn J  ||      rJ |dd }|j                  |j                  cxk(  rdk(  rn J  ||      rJ |d d  } ||      dk(  sJ |dd  } ||      dk(  sJ |d d } ||      dk(  sJ |d d } ||      dk(  sJ |dd  } ||      dk(  sJ |dd } ||      dk(  sJ |dd } ||      dk(  sJ |dd }|j                  |j                  cxk(  rdk(  rn J  ||      rJ |dd }|j                  |j                  cxk(  rdk(  rn J  ||      rJ |dd }|d   j                  dk(  sJ |d d  } ||      d	k(  sJ |d d
 } ||      dk(  sJ |dd  } ||      dk(  sJ |d d } ||      dk(  sJ |dd  } ||      dk(  sJ |dd
 } ||      dk(  sJ |dd } ||      dk(  sJ |dd } ||      d	k(  sJ |dd }|j                  |j                  cxk(  rdk(  rn J  ||      rJ y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   |xY w)NGive it back! He pleaded.r   Giver   r0   c                 2    dj                  d | D              S )N/c              3   4   K   | ]  }|j                     y wr   )rB   ).0rl   s     r+   	<genexpr>z7test_doc_api_getitem.<locals>.to_str.<locals>.<genexpr>.  s     5u

5s   )r   r   s    r+   to_strz$test_doc_api_getitem.<locals>.to_str-  s    xx5555rI   r   r#   z	it/back/!r   r6   z
He/pleadedzback/!r;   zGive/it/back/!/He/pleaded/.zHe/pleaded/.zGive/it/back/!2   ii(   r<   itzit/backr   back)	rB   pytestraises
IndexErrorrS   
ValueErrorstartr   orth_)en_tokenizerrB   tokensr   rZ   subspans         r+   test_doc_api_getitemr  %  sM   &D$F!9>>V###":??c!!!	z	" s6{6 !A;Dd|!A;D$<;&&&!Aa%=D$<;&&&	z	" q1u	z	" q2v "Q<D$<<'''!B<D$<<'''"R=D$<8###!A;D::&Q&;;vd|;;!B<D::&Q&;;vd|;;!9D$<8888!":D$<>)))"1:D$<++++#2;D$<++++"#;D$<>)))!B<D$<>)))#a=D$<++++#c?D::&Q&;;vd|;;"R=D::&Q&;;vd|;;!A;D7==D   1gG'?k)))2AhG'?i'''12hG'?h&&&3BiG'?i'''23iG'?h&&&1QiG'?f$$$2bkG'?f$$$3rlG'?k)))2clG==GKK,1,DDVG_DD_G   s$   M5	M	M"MM"M,r   z Give it back! He pleaded. c                     | |      }d|d   _         d|d   _        |j                  j                  d   ddfg|_        d|d   _        d|d   _        t        |j                        j                  |j                               }|j                  |j                  k(  sJ |D cg c]  }|j                   c}|D cg c]  }|j                   c}k(  sJ |D cg c]  }|j                   c}|D cg c]  }|j                   c}k(  sJ |d   j                   dk(  sJ |d   j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  dk(  sJ t        |j                        j                  |j                  dg	      dg	      }|j                  |j                  k(  sJ |D cg c]  }|j                   c}|D cg c]  }|j                   c}k(  sJ |D cg c]  }|j                   c}|D cg c]  }|j                   c}k(  sJ t        |j                        j                  |j                  d
g	      d
g	      }|j                  |j                  k(  sJ |D cg c]  }|j                   c}|D cg c]  }|j                   c}k(  sJ |D cg c]  }|j                   c}|D cg c]  }|j                   c}k(  sJ d }|j                         }t        j                  t              5  ||j                  d<   |j                         }d d d        y c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w # 1 sw Y   y xY w)Nlemmar   normr8   r   	ent_kb_ident_idtensorexclude	sentimentc                      y)Nzhello!r   )d1d2s     r+   
inner_funcz*test_doc_api_serialize.<locals>.inner_func  s    rI   
similarity)lemma_norm_r=   r>   r?   
ent_kb_id_ent_id_r   
from_bytesto_bytesrB   orthr   warnsUserWarning
user_hooks)r   rB   r   
new_tokensr*   r  r   s          r+   test_doc_api_serializer  p  s    $FF1IF1IOLL((3Q:;FK&F1I F1IV\\"--foo.?@J;;*//)))"#qAFF#
'C1'CCCC"#qAFF#
'C1'CCCCa=7***a=&(((a=##{222a=  H,,,V\\"--
+hZ . J ;;*//)))"#qAFF#
'C1'CCCC"#qAFF#
'C1'CCCCV\\"--. . J ;;*//)))"#qAFF#
'C1'CCCC"#qAFF#
'C1'CCCC 	A	k	" *4,'OO 3 $'C#'C $'C#'C $'C#'C sT   $L5=L:L?4M5M	M,MM6MM"-M'M, M11M:c                    d} | |      }t        |j                        dk(  sJ |j                  j                  d   ddfg|_        t        t	        |j                              dk(  sJ |D cg c]  }|j
                   c}g dk(  sJ |j                  d   j                  dk(  sJ |j                  d   j                  dk(  sJ |j                  d   j                  dk(  sJ y c c}w )Nz#I use goggle chrone to surf the webr   r8   r   r#   r   )r   r   rP   r   r   r   r   r   )	rS   r?   r=   r>   r   ent_ioblabel_r   r   )r   rB   r   r*   s       r+   test_doc_api_set_entsr    s    0D$Fv{{q   LL((3Q:;FKtFKK !Q&&&%&!AII&*BBBB;;q>  I---;;q>1$$$;;q>""" 's   .C&c                 ^     | d      }t        |j                        }t        |      dk(  sJ y )Nr_   r   )r   r   rS   )r   r)   r   s      r+   test_doc_api_sents_empty_stringr!    s)    
r
COEu:??rI   c                 h   d}g d} | |      }t        |j                  |D cg c]  }|j                   c}|      }g }|j                  D ]f  }t	        |      dkD  r6|d   j
                  dvr%|dd  }t	        |      dkD  r|d   j
                  dvr%t	        |      dkD  sV|j                  |       h |j                         5 }|D ]M  }|j                  j                  |j                  |j                  j                  d}	|j                  ||	       O 	 d d d        y c c}w # 1 sw Y   y xY w)	Nu   67% of black households are single parent 

72% of all black babies born out of wedlock 

50% of all black kids don’t finish high school)nummodrz   r|   amodr}   r   r$  attrr_   r#  apposr|   r   r$  r}   aclr|   r|   r}   r_   r#  rz   r|   r   r$  r}   auxr   r{   r$  r   )r    r&   r   r   )advmodr$  compound)tagr  ent_typer   )r   r=   rB   noun_chunksrS   r   r   r@   rootr   	ent_type_rA   )
r   rB   r&   r   r*   r)   npsnprC   r   s
             r+   test_doc_api_runtime_errorr3    s'    _DiD $F
fll6":a166":
FC
Coo "gkbejj0NNABB "gkbejj0NNr7Q;JJrN	
 
	 /[ 	/Bww||GG--E
 b.	// / #;/ /s   D#
AD((D1c                    g d}g d}dgt        |      z  }t        | |||      }|d   j                  dk(  sJ |d   j                  D cg c]  }|j                   }}|g dk(  sJ |d   j                  j                  dk(  sJ y	c c}w )
zHTest for bug occurring from Unshift action, causing incorrect right edge)Ihaveproposedtomyselfrt   rN   thesakeofsuchasliveunderr:  
governmentr<  r:  Romansrt   r8  	translatethosebooksintor:  Greektonguer0   )r   r   r   r   rP   r      r7   r6   r7      r7   rJ           rL        rK  rI  r      rI  rI     rQ     r   r"   r$   r6   rN   )rN   r:  r;  r<  r=  r>  r?  r@  r:  rA  r<  r:  rB  rt   rt   N)rS   r   rB   subtree
right_edge)r(   r    r%   r&   r)   r   rS  s          r+   test_doc_api_right_edgerU    s    E uE7SZD
he5t
<Cq6;;%"1v~~.!qvv.G.  C  C  C  Cq6!!S(((	 /s   B	c                      t               } | j                  d       | j                  dt        j                  ddgd             t        | dg	      }|j                  sJ y )
Nr   )widthkitten        g       @frQ   )vectorr5   )r   reset_vectors
set_vectorrT   asarrayr   
has_vector)r=   r)   s     r+   test_doc_api_has_vectorr`    sV    GE	a 	XemmS#Jc&JK
eH:
&C>>>rI   c                     t        t               dg      } | j                  | d         dk(  sJ | j                  | j                  d         dk(  sJ t        | j                  g d      }t	        j
                  t              5  | j                  |d d       dk(  sJ | j                  |      dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr   r5   r   g      ?rG   r   rY  )r   r   r  r=   r   r  r  )r)   r   s     r+   test_doc_api_similarity_matchrb    s    
egcU
#C>>#a&!S(((>>#))C.)S000syy0D	k	" +~~d2Ah'3...~~d#s***+ + +s   0B??Czwords,heads,lca_matrix)r:  lazydogslept)r   r   rP   rP   )r   r   r   rP   )r   r   r   rP   )r   r   r   rP   )rP   rP   rP   rP   )	Therc  rd  re  r0   rf  quickfoxjumped)	r   r   rP   rP   rP   r<   r<   r7   r7   )	r   r   r   rP   rP   r   r   r   r   )	r   r   r   rP   rP   r   r   r   r   )	r   r   r   rP   rP   r   r   r   r   )	rP   rP   rP   rP   rP   r   r   r   r   )	rP   rP   rP   rP   r#   r   r   r   r   )	r   r   r   r   r   r;   r<   r<   r7   )	r   r   r   r   r   r<   r6   r<   r7   )	r   r   r   r   r   r<   r<   r<   r7   )	r   r   r   r   r   r7   r7   r7   r7   c           	          t        | ||dgt        |      z        }|j                         }||k(  j                         sJ |d   dk(  sJ |d   dk(  sJ |d   dk(  sJ y )Nr"   )r%   r&   )r   r   r   )r   r   r   )r   r   )r   rS   rW   rX   )r(   r    r%   
lca_matrixr)   lcas         r+   test_lowest_common_ancestorrm    st    8 hU%3u:1E
FC



C:""$$$t9>>t9>>t9>>rI   c                    g d}t        | |      }|j                  d      rJ t        |ddd      g|_        |j                  d      sJ t	        j
                  ddgddgddgd	dgd	d
ggd      }t        | |      j                  t        t        g|      }|j                  d      sJ t        |       j                  |j                               }|j                  d      sJ y )N)r5  r?  inNewYorkr5   r   rP   r;   GPEr9   r   i  r   r   rQ   )r   has_annotationr   r?   rT   rU   r   r   r   r  r  )r(   r    r)   arrnew_docs        r+   test_doc_is_neredrv    s    .E
he
$C!!),,,S!Qe,-CHi(((
++1v1v1vQx#qB(
SC
he
$
/
/70CS
ICi((((m&&s||~6G!!),,,rI   c                    g d}g d}g d}t        | |||      }t        t        g}|j                  |      }t        | |      }|j	                  ||       |j                         }|j                  |      }t        j                         5  t        j                  d       |j	                  ||       d d d        t        g}|j                  |      }t        | |      }|j	                  ||       |D cg c]  }|j                   c}|D cg c]  }|j                   c}k(  sJ |j                  d      rJ t        t        g}|j                  |      }t        | |      }|j	                  ||       |D cg c]  }|j                   c}|D cg c]  }|j                   c}k(  sJ |j                  d      sJ y # 1 sw Y   xY wc c}w c c}w c c}w c c}w )N)
r5  r?  ro  rp  rq  r0   r5  likecatsr0   )
r   r   r   r   r   r   r6   r6   r6   r6   )
r   r"   r"   r"   r"   r"   r   r"   r"   r"   r$   r5   errorr   )r   r   r	   to_arrayr   _get_array_attrswarningscatch_warningssimplefilterr'   rs  r   )	r(   r    r%   r&   r)   r   rt  ru  r*   s	            r+   test_doc_from_array_sent_startsr     s   ME*ESD
he5t
<CE
,,u
C(%(Guc"  "E
,,u
C		 	 	" 'g&5#&' LE
,,u
C(%(Guc"%()AOO)w-O!aoo-OOOO%%e,,,3KE
,,u
C(%(Guc"%()AOO)w-O!aoo-OOOO!!%(((!' ' *-O *-Os$   (G8GG>GG!Gc                    g d}g d}t        | ||      }t        g}|j                  |      }t        | |      }|j                  ||       |D cg c]  }t	        |j
                         c}|k(  sJ |D cg c]  }t	        |j
                         c}|D cg c]  }t	        |j
                         c}k(  sJ y c c}w c c}w c c}w )N)r5  r?  ro  rp  rq  r0   )zFeat1=AzFeat1=BzFeat1=CzFeat1=A|Feat2=DzFeat2=EzFeat3=F)r    morphsr5   )r   r   r{  r   r   morph)r(   r    r  r)   r   rt  ru  r*   s           r+   test_doc_from_array_morphr  B  s    3EWF
heF
3CGE
,,u
C(%(Guc"")*QCL*f444"%&QCL&*IA3qww<*IIII +&*Is   C :CC
r   c                 R   g d}|D cg c]  }t        |      s| }}d}|D cg c]
  } | |       }}|d   dd g|d   j                  d<   |d   dd g|d   j                  d<   |d   dd g|d   j                  d<   t        |d   dd j                  |d   dd j                  |d   dd j                  g      } ||      }	t	        j
                  dd	
       d|d   d   j                  _        d|d   d   j                  _        t        j                  g       J |	t        j                  |	g      usJ t        |	      t        t        j                  |	g            k(  sJ t        j                  t              5  t        j                  ||	gz          d d d        t        j                  |      }
t        |      t        t        |
j                              k(  sJ t        |
j                        t        |d         t        |d         z   kD  sJ |
j                  dj!                  |D cg c]  }|j#                          c}      k(  sJ |
t        |d         dz
     }|j                  dk(  rt%        |j&                        sJ |D cg c]  }|D ]  }|  }}}t        |
      t        |      k(  sJ t        |d         dz   |d   j)                  d      z   }|
d   j                  j                  du sJ |
d   j*                  |k(  sJ |
d   j                  j                  du sJ t-        |
dd D cg c]  }|j                  j                   c}      rJ d|
j                  v sJ |t        |
j                  d   D cg c]  }|j                   c}      k(  sJ t%        |
d   j&                        sJ t        j                  |d	      }
t        |      t        t        |
j                              k(  sJ t        |
j                        t/        d |D              k(  sJ |
j                  dj!                  |      k(  sJ |
t        |d         dz
     }|j                  dk(  rt%        |j&                        rJ |D cg c]  }|D ]  }|  }}}t        |
      t        |      k(  sJ t        |d         dz   |d   j)                  d      z   }|
d   j*                  |k(  sJ d|
j                  v sJ |t        |
j                  d   D cg c]  }|j                   c}      k(  sJ t%        |
d   j&                        sJ t        j                  |g d      }
t        |
j                        t        |d         t        |d         z   kD  sJ |
j                  dj!                  |D cg c]  }|j#                          c}      k(  sJ |
t        |d         dz
     }|j                  dk(  rt%        |j&                        sJ |D cg c]  }|D ]  }|  }}}t        |
      t        |      k(  sJ t        |d         dz   |d   j)                  d      z   }|
d   j*                  |k(  sJ d|
j                  v sJ |t        |
j                  d   D cg c]  }|j                   c}      k(  sJ t        j                  |dg      }
d|
j                  vsJ t        j                  |dg      }
|
j0                  i k(  sJ t        j                   | d      gdz        }|D cg c]
  } | |       }}t        j                  |      }
d|
j                  vsJ |D ]  }g |j                  d<    t        j                  |      }
d|
j                  v sJ t        |
j                  d         dk(  sJ t3               }|D ];  }|j5                  |D cg c]  }t        |j                        dg c}      |_        = t        j                  |      }
t9        |j;                  |
j6                        |j;                  |j<                  j?                  |D cg c]  }t        |      s|j6                   c}                   t        j                  |dg      }
|
j6                  j@                  dk(  sJ y c c}w c c}w # 1 sw Y   AxY wc c}w c c}}w c c}w c c}w c c}}w c c}w c c}w c c}}w c c}w c c}w c c}w c c}w )N)Merging the docs is fun.r_   zThey don't think alike. r_   zAnother doc.zWie war die Frage?r   r   r#   groupr   is_ambiguousFr   TrP   r   r0   thinkry   r7   rJ  )ensure_whitespacec              3   2   K   | ]  }t        |        y wr   )rS   )r   r*   s     r+   r   z)test_doc_api_from_docs.<locals>.<genexpr>  s     !;Q#a&!;s   r_   )r  lengthr   r-  spansr  	user_data
   rY  r  )r   )!rS   r  sortedrB   r   r   r   r  r   	from_docsr   r   r   r   r   r   r   stripboolwhitespace_indexidxanysumr  r   r^  r  r   to_numpyxpvstackshape)r   de_tokenizeren_textsr*   en_texts_without_emptyde_textrB   en_docsspan_group_textsde_docm_docp_tokenr)   en_docs_tokens	think_idxsopss                    r+   test_doc_api_from_docsr  P  sb   H *2<ASVa<<"G.67d|D!7G7!(Aa 1GAJW!(Aa 1GAJW!(Aa 1GAJW	Aa		wqz!A33WQZ!_5I5IJ '"F	6#'GAJqMOO #'GAJqMOO ==$$$x0000v;#cmmVH56666	z	" *g()* MM'"E%&#d5;;.?*@@@@uzz?S!-HQK0@@@@@::6L"M1779"MNNNNC
Oa'(G<<34(;(;#<<<#*8CC8qa8a8N8u:^,,,,HQK 1$x{'8'8'AAI8::""d***8<<9$$$8::""d***eAaj9ACC$$9:::ekk!!!vu{{77K&L!qvv&LMMMMb	%%&&&MM'U;E%&#d5;;.?*@@@@uzz?c!;(!;;;;;::!78888C
Oa'(G<<3tG,?,?'@@@#*8CC8qa8a8N8u:^,,,,HQK 1$x{'8'8'AAI8<<9$$$ekk!!!vu{{77K&L!qvv&LMMMMb	%%&&&MM')CDEuzz?S!-HQK0@@@@@::6L"M1779"MNNNNC
Oa'(G<<34(;(;#<<<#*8CC8qa8a8N8u:^,,,,HQK 1$x{'8'8'AAI8<<9$$$ekk!!!vu{{77K&L!qvv&LMMMM MM'G95E%++%%% MM'K=9E??b    --b)*R/
0C /77d|D!7G7MM'"E%++%%%  		' MM'"Eekk!!!u{{7#$))) 
C D[[c!B3qvv;"4!BC
DMM'"EU\\"SVV]]'#N3SXCJJ#NOP MM'H:6E<<%%%C =7* * #N 9 :&L 9
 'M #N 9
 'M 8 "C $Osi   cccc6c*
c/c5c:
c?!d
d

"d/d
d-d
1d$d$c'c                     ddg}|D cg c]
  } | |       }}d|d   _         t        |d   ddd      f|d   _         t        j                  |      }t	        |j                         dk(  sJ y c c}w )Nr  zThey don't think alike.r   r   r   foor9   )r?   r   r   r  rS   )r   textsr*   r   r)   s        r+   test_doc_api_from_docs_entsr    su    ')BCE%*+LO+D+DGLa!Qe46DGL
--
Csxx=A	 ,s   A/c                    t        | ddg      }|j                  dk(  sJ |j                  | j                  d   k(  sJ |d   j                  dk(  sJ |d   j                  | j                  d   k(  sJ t	               } |d      }|j                  dk(  sJ |j                  | j                  d   k(  sJ |d   j                  dk(  sJ |d   j                  | j                  d   k(  sJ y )NHelloworldr5   enr   zHello world)r   lang_langr>   r   )r(   r)   re   s      r+   test_doc_langr    s    
hw0
1C9988x''----q6<<4q6;;(**40000
)C
m
C9988x''----q6<<4q6;;(**40000rI   c                     t        | ddg      d   }t        |j                  t              sJ |j                  j                  |j                  k(  sJ | |j
                     |j                  k(  sJ y)z%Test that tokens expose their lexeme.r  r  r5   r   N)r   r   lexr   rB   r  )r(   rl   s     r+   test_token_lexemer    sa    ' 23A6Eeii(((99>>UZZ'''EJJ599,,,rI   c                    t        | ddg      }d}|D ]*  }|j                  |      rJ |j                  |d      s*J  d|d   _        d	|d   _        |d   j	                  d
       d|d   _        d|d   _        |d   |d   _        |j                  t        |ddd      gd       |D ]*  }|j                  |      sJ |j                  |d      s*J  d|d   _        d	|d   _        |d   j	                  d       d|d   _        d|d   _        t        |ddd      g|_
        |D ]*  }|j                  |      sJ |j                  |d      r*J  y )Nr  r  r5   )r   r   r   LEMMAr   r	   r   r   Trequire_completer]   r   XFeat=Valr   r"   r   HELLOr9   missingr   r_   r   )r   rs  r   r   	set_morphr  r   r   set_entsr   r?   r(   r)   r   r%  s       r+   test_has_annotationr    s   
hw0
1CRE C%%d+++%%dT%BBBC CFKCFKFZ CFMCFKa&CFKLL$sAq019LE C!!$'''%%dT%BBBC CFKCFKFRCFMCFKS!Qg./CH ?!!$'''!!$!>>>?rI   c                 h   t        | g d      }d}|D ]*  }|j                  |      rJ |j                  |d      s*J  d|d   _        |D ]*  }|j                  |      sJ |j                  |d      s*J  d|d   _        |D ]*  }|j                  |      sJ |j                  |d      r*J  y )	N)r  	beautifulr  r5   )r   IS_SENT_STARTIS_SENT_ENDTr  Fr   r   )r   rs  r'   r  s       r+   test_has_annotation_sentsr    s    
h=
>C:E C%%d+++%%dT%BBBC !CF C!!$'''%%dT%BBBC !CF ?!!$'''!!$!>>>?rI   c                     | d      }t        j                         5  |j                   d d d        t        j                         5  |j                   d d d        t        j                         5  |j                   d d d        t        j                         5  |j
                   d d d        y # 1 sw Y   xY w# 1 sw Y   hxY w# 1 sw Y   KxY w# 1 sw Y   y xY w)NrM   )r   deprecated_call	is_tagged	is_parsedis_neredis_sentencedr   r)   s     r+   test_is_flags_deprecatedr    s    
v
C				! 				! 				! 				!      s/   B.B:/CC.B7:CCCc           	      	    | d      }|j                  t        |ddd      t        |ddd      g       |D cg c]  }|j                   c}g dk(  sJ |D cg c]  }|j                   c}g dk(  sJ  | d      }|j                  t        |ddd      t        |ddd      g       |j                  t        |dd	d
      gd       |D cg c]  }|j                   c}g dk(  sJ |D cg c]  }|j                   c}g dk(  sJ  | d      }|j                  t        |ddd      t        |ddd      g|dd g       |D cg c]  }|j                   c}g dk(  sJ |D cg c]  }|j                   c}g dk(  sJ  | d      }|j                  t        |ddd      t        |ddd      g|dd gd       |D cg c]  }|j                   c}g dk(  sJ |D cg c]  }|j                   c}g dk(  sJ  | d      }|j                  g |dd	 |dd gd       |D cg c]  }|j                   c}g dk(  sJ |D cg c]  }|j                   c}g dk(  sJ |j                  t               k(  sJ t        |ddd      g|_        |D cg c]  }|j                   c}g dk(  sJ |j                  g |dd gd       |D cg c]  }|j                   c}g dk(  sJ  | d      }|j                  t        |ddd      g|dd	 g|d	d g|dd gd       |D cg c]  }|j                   c}g dk(  sJ |D cg c]  }|j                   c}g dk(  sJ  | d      }t        j                  t              5  |j                  g |dd	        d d d        t        j                  t              5  |j                  g |dd	 gd        d d d        t        j                  t              5  |j                  g |dd	 g|dd	 g!       d d d        y c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)"Nz	a b c d er   r   r  rP   rJ  )rP   rP   r   r   r   )r  rJ  rJ  r   r   r   rK  
unmodifiedr   )rP   r   rP   r   r   )rK  rK  rJ  r   r   r#   r;   )r  )rP   rP   r   r   r   r  )outsider   )rP   rP   r   r   r   )blockedr   )r   rP   r   rP   rP   )r   r   r   r   r   ENT)r   r   r   rP   r   )r   r   r   rP   rP   )r  r  r  r   )rP   rP   r   r   r   )r  r   r   r   r   none)r  r   )r  r  )	r  r   r  r,  r?   tupler   r   r   )r   r)   r*   s      r+   test_doc_set_entsr    sw   
{
#CLL$sAq"%tCAr':;<"#!AII#666 #$1AJJ$(:::: {
#CLL$sAq"%tCAr':;<LL$sAq"%&L="#!AII#666 #$1AJJ$(:::: {
#CLL$sAq"%tCAr':;c!AhZLP"#!AII#666 #$1AJJ$(:::: {
#CLL	c1a	d31b12Qq
  
  ##!AII#666 #$1AJJ$(:::: {
#CLLc!AhAa1<LH"#!AII#666 #$1AJJ$77788uw S!Q&'CH"#!AII#666LLc!AhZL>"#!AII#666 {
#CLL	c1a	Qq
Qq
Qq
    ##!AII#666 #$1AJJ$(8888
{
#C	z	" +RQq*+ 
z	" =R#a(V<= 
z	" AR#a(c!AhZ@A Aw $$ $$
 $$ $$
 $$
 $# $$+ += =A Asw   P!P&P+.P0P5)P:P?%Q*Q	QQQ*QQ"	Q'Q3;Q?'Q03Q<?Rc                     g d} t        t               |       }d|j                  j                  j	                  d      ddfg|_        |j
                  D cg c]  }|j                   c}ddgk(  sJ t               }d|j                  j	                  d      ddfg}g d}t        || |	      }|j
                  D cg c]  }|j                   c}ddgk(  sJ y
c c}w c c}w )zaTest that both strings and integers can be used to set entities in
    tuple format via doc.ents.r   r   r   r   er5   )r  r   r   WORLDrP   r;   r  )zB-HELLOzI-HELLOOzB-WORLDzI-WORLDr    r?   N)r   r   r=   r>   r   r?   r  )r    r)   r  r=   r?   s        r+   test_doc_ents_setterr  _  s     &E
egU
#C#))"3"3"7"7"@!Q!GHCH!hh'AHH'GW+====GEemm//8!Q?@D<D
e5t
,C!hh'AHH'GW+==== (
 (s   C>C!c                     | d      } | d      } |d      }|d   j                   |d   _         |d   j                   j                  dk(  sJ |d   j                   j                  dk(  sJ |d   j                  d       |d   j                   |d   _         |d   j                   |d   j                   k(  sJ |d   j                   |d   _         |d   j                   |d   j                   k(  sJ |d   j                  d       t        j                  t
              5  |d   j                   |d   _         d d d        y # 1 sw Y   y xY w)Nza bzc dr   r   r  z
Feat2=Val2)r  keyr  r   r   r   )r   r  doc1doc1br   s        r+   test_doc_morph_setterr  m  s-   DED GMMDGM7==!!!7==!!! 	Gj!GMMDGM7==DGMM))) !W]]E!HN7==E!HNN***Gl# 
z	" &QQ& & &s   D>>Ec                  X   g d} dgt        |       z  }t        t               | |      }|j                  dk(  sJ g d}t        t               | |      }t        |j                        dk(  sJ g d}t        t               | |      }t        |j                        dk(  sJ g d	}t        t               | |      }t        |j                        dk(  sJ g d
}t        t               | |      }t        |j                        dk(  sJ g d}t	        j
                  t              5  t        t               | |      }ddd       g d}t	        j
                  t              5  t        t               | |      }ddd       g d}t	        j
                  t              5  t        t               | |      }ddd       g d}t	        j
                  t              5  t        t               | |      }ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   ^xY w# 1 sw Y   yxY w)z2Test ents validation/normalization in Doc.__init__r  r  r  r   )B-PERSONI-PERSONr  r  r  r   )r  r  r  r  I-GPErP   )r  r  r  Nr  )r_   r  r  r  r  )zQ-PERSONr  r  r  r  N)OPERSONr  r  r  r  )r  B-r  r  r  )r   r  r  r  r  )rS   r   r   r?   r   r   r   )r    r?   r)   s      r+   test_doc_init_iobr    s   %E53u:D
egU
.C88r>>@D
egU
.Csxx=A=D
egU
.Csxx=A 8D
egU
.Csxx=A 9D
egU
.Csxx=A >D	z	" 3%'T23 =D	z	" 3%'T23 1D	z	" 3%'T23 /D	z	" 3%'T23 33 3
3 3
3 3
3 3s0   (G<$H HH <HHH H)c                 6    | d      }t        |ddd      t        |ddd      g}|j                         5 }|D ]  }|j                  |        	 d d d        t        j                  t
              5  ||_        d d d        y # 1 sw Y   3xY w# 1 sw Y   y xY w)N/Some text about Colombia and the Czech RepublicrP   r#   rr  r9   r6   r7   )r   r@   rA   r   r   r   r?   )r   r)   r  rC   rZ   s        r+   test_doc_set_ents_invalid_spansr    s    
H
IC#q!5)4Q+GHE		 $[ 	$Dd#	$$ 
z	"  $ $ s   B2BBBc                      d} t               } ||       }t        j                  t              5  t	        |j
                        }ddd       y# 1 sw Y   yxY w)zNTest that a language without noun_chunk iterator, throws a NotImplementedErroru   Může data vytvářet a spravovat, ale především je dokáže analyzovat, najít v nich nové vztahy a vše přehledně vizualizovat.N)r   r   r   NotImplementedErrorr   r.  )rB   re   r)   r   s       r+   $test_doc_noun_chunks_not_implementedr    sJ     VD
/C
d)C	*	+ "!" " "s   AAc                 
    | d      }t        |ddd      g|j                  d<   d|j                  v sJ d|j                  vsJ t        |j                  d         dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d   j	                  |dd        t        |j                  d         d	k(  sJ |j                  d   d   j
                  d
k(  sJ |j                  d   D cg c]  }|j
                   c}dd
gk(  sJ |j                  d   j                  rJ t        |ddd      t        |ddd      g|_        |j                  d   j                  |j                         t        |j                  d         dk(  sJ |j                  d   D cg c]  }|j                   c}g dk(  sJ |j                  d   j                  sJ |j                  d= d|j                  vsJ y c c}w c c}w )Nr  rP   r#   byer9   hir   r   r   zSome text aboutColombiarr  r6   r7   )r  r_   rr  rr  )	r   r  rS   r  r   rB   has_overlapr?   extend)r   r)   rZ   s      r+   test_span_groupsr    s   
H
ICCAU34CIIdO399		!!!syy1$$$99T?1$$---IIdO3q8$syy1$$$99T?1""&7777"%))D/2$DII2zCT6UUUUyy****S!Qe,d31E.JKCHIIdO388$syy1$$$$'IIdO4DDKK48QQQQ99T?&&&&		$syy    3
 5s   4G;%H c                      | d      }t        j                  |      |j                  j                  k(  sJ |j	                         }t        j                  |      |j                  j                  k(  sJ y )Nr  )weakrefrefr  doc_refcopy)r   r  r   s      r+   test_doc_spans_copyr    sZ    IJD;;t

 2 222299;D;;t

 2 2222rI   c           
          | d      }|j                   j                  d       t        |j                   d         dk(  sJ |j                   j                  d|dd g       t        |j                   d         dk(  sJ |j                   j                  dt        ||dd |dd g	             t        |j                   d         dk(  sJ y )
Nr  key1r   key2r   r   key3r   )r  )r  
setdefaultrS   r   r  s     r+   test_doc_spans_setdefaultr    s    
H
ICII syy !Q&&&II#a(4syy !Q&&&II3s1QxQq>R)STsyy !Q&&&rI   )Sr}  r  rT   r   numpy.testingr   	thinc.apir   r   spacy.attrsr   r   r   r	   r
   r   r   r   r   spacy.lang.enr   spacy.lang.xxr   spacy.languager   spacy.lexemer   spacy.tokensr   r   r   r   spacy.vocabr   test_underscorer   r,   markissuerE   rH   r[   ra   parametrizerf   rm   r   r   factoryr   r   r   r   r  r  r  r!  r3  rU  r`  rb  rU   rm  rv  r  r  usefixturesr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   rI   r+   <module>r     s       , /
 
 
 " ' #  4 4  -H" 4* * 4
% 
% 43 3* 5( ( !=>g}%=>4  ? ? 	 45 	5 4$& $&N 4!) !)H )  * 46 6 4 , "9!:; <HEV 
(*GH$$N	#/4)*+  ,EKK|\<PQ	
 R'EKK333334444
	
676-)DJ +,i& -i&X1-?>?*	@AF>&4+3\"!*3'rI   