
    iJ                        d dl Z d dlmZ d dlmZmZ d dlmZ d Zd Z	d Z
d Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Ze j0                  j3                  dddiddiddidgg      d        Zd Zd Zd Zd Zy)     N)LEMMA)DocToken)Vocabc                    d}ddddd} | |      }t        |      dk(  sJ |j                         5 }|j                  |dd	 |
       |j                  |d	d |
       d d d        t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  dk(  sJ t        |d   j                        dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j
                  dk(  sJ t        |d   j                        dk(  sJ |d   j                  dk(  sJ y # 1 sw Y   xY w)N-WKRO played songs by the beach boys all nightNAMEDr   TYPEzNumber=Plur)taglemmaent_typemorph	         attrs   the beach boyszthe beach boys    z	all night)	len
retokenizemergetexttext_with_wstag_lemma_strr   )en_tokenizerr   r   docretokenizers        v/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/spacy/tests/doc/test_retokenize_merge.pytest_doc_retokenize_merger#      s   :D	E t
Cs8q==		 1[#a(%0#a(%01 s8q==q6;;****q6"3333q6;;'!!!q6==G###s1v||---q6;;+%%%q6+---q6;;'!!!s1v||---q6==G###1 1s   -EEc                    d}dddd} | |      }t        |      dk(  sJ |j                         5 }|j                  |dd |	       d
d
d
       |D ]  }|j                  |j                  j                  k  r$|t        |j                  j                        v rHJ |j                  |j                  j                  kD  sn|t        |j                  j                        v rJ  y
# 1 sw Y   xY w)z3Test that attachments work correctly after merging.r   r	   r   r
   r   r   r   r   r   r   r   N)r   r   r   iheadlistleftsrights)r   r   r   r    r!   words         r"   "test_doc_retokenize_merge_childrenr,   "   s    :Dg6BE
t
Cs8q==		 1[#a(%01 266DIIKK4		0000VVdiikk!4		 0 01111	21 1s   C&&C/c                     d} | |      }|j                         5 }|j                  |dd ddd       |j                  |dd	 ddd       d d d        y # 1 sw Y   y xY w)
Nz through North and South Carolina   r    ORGr   r   r         )r   r   r   r   r    r!   s       r"   test_doc_retokenize_merge_hangr5   1   sr    -D
t
C		 L[#a(BE*JK#a(BE*JKL L Ls   3AA c                      | d      }|j                         5 }|j                  |dd        d d d        t        |      dk(  sJ |d   j                  dk(  sJ y # 1 sw Y   .xY w)Nr   r   r   r   )r   r   r   r   )r   r    r!   s      r"   test_doc_retokenize_retokenizerr7   9   si    
F
GC		 $[#a(#$s8q==q6;;****$ $s   AA$c                 d    | d      }t         dd|j                  j                  d   i}|j                         5 }|j	                  |dd |       d d d        t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ y # 1 sw Y   VxY w)	Nr   boysENT_TYPEr0   r   r   r   r   )	r   vocabstringsr   r   r   r   r   	ent_type_)r   r    r   r!   s       r"   %test_doc_retokenize_retokenizer_attrsr>   A   s    
F
GCFJ		(9(9%(@AE		 1[#a(%01s8q==q6;;****q6==F"""q6u$$$1 1s   B&&B/c                 |    | d      }t        d |D              rJ |j                         5 }|j                  |dd ddd       d	d	d	       |d   j                  d
k(  sJ |d   j                  dk(  sJ |d   j
                  sJ t        |j                  d
g      }|d   j
                  sJ y	# 1 sw Y   lxY w)z8Test that lexical attributes can be changed (see #2390).zWKRO played beach boys songsc              3   4   K   | ]  }|j                     y wNis_stop).0tokens     r"   	<genexpr>z0test_doc_retokenize_lex_attrs.<locals>.<genexpr>P   s     2U5==2   r3   r   r9   T)r   IS_STOPr   Nz
beach boyswordsr   )anyr   r   r   r   rC   r   r;   )r   r    r!   new_docs       r"   test_doc_retokenize_lex_attrsrM   M   s    
5
6C2c2222		 N[#a(Ft*LMNq6;;,&&&q6==F"""q6>>>#))L>2G1:N Ns   B22B;c                 p   d}g d}dgt        |      z  } | |      }t        |j                  |D cg c]  }|j                   c}||      }t        |      dk(  sJ |d   j                  j                  dk(  sJ |d   j                  j                  d	k(  sJ |j                         5 }d
ddd}|j                  |dd |       d d d        t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  j                  d	k(  sJ |d   j                  dk(  sJ y c c}w # 1 sw Y   exY w)NLos Angeles start.r2   r3   r3   r3   deprJ   headsdepsr   r   Angelesr2   startNNPLos AngelesGPEr%   r3   r   r.   )r   r   r;   r   r'   r   r   r=   )	r   r   rS   rT   tokenstr    r!   r   s	            r"   &test_doc_retokenize_spans_merge_tokensr\   Z   s8   DE7SZD$F
fll6":a166":%d
SCs8q==q6;;y(((q6;;w&&&		 1[5I#a(%01 s8q==q6;;-'''q6;;w&&&q6u$$$ #;1 1s   D'
+D,,D5c           	      |   g d}|D cg c]  }|j                          }}g d}dgt        |      z  }g d}g d}t        | ||||||      }t        |      dk(  sJ |d   j                  d	k(  sJ |d   j                  d
k(  sJ |d   j
                  dk(  sJ |d   j                  dk(  sJ |j                         5 }	|	j                  |dd        d d d        t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  dk(  sJ t        | ||||||      }t        |      dk(  sJ |d   j                  d	k(  sJ |d   j                  d
k(  sJ |d   j
                  dk(  sJ |d   j                  dk(  sJ |j                         5 }	|	j                  |dd        |	j                  |dd        d d d        t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j
                  dk(  sJ |d   j                  dk(  sJ y c c}w # 1 sw Y   xY w# 1 sw Y   xY w)N)TheplayersrV   .rP   rQ   )DTNNVBZr`   )DETNOUNVERBPUNCT)rJ   tagsposrS   rT   lemmasr   r   r^   ra   rd   ther3   r.   zThe playersrb   re   zthe playersr2   zstart .rc   rf   )	lowerr   r   r   r   pos_r   r   r   )
en_vocabrJ   r[   rj   rS   rT   rh   ri   r    r!   s
             r"   4test_doc_retokenize_spans_merge_tokens_default_attrsro   l   s   ,E!&'Aaggi'F'E7SZD#D
*C
DcTRXC s8q==q6;;%q6;;$q6;;%q6==E!!!		 $[#a(#$s8q==q6;;-'''q6;;$q6;;&   q6==M)))
DcTRXC s8q==q6;;%q6;;$q6;;%q6==E!!!		 $[#a(##a(#$ s8q==q6;;-'''q6;;$q6;;&   q6==M)))q6;;)###q6;;%q6;;&   q6==I%%%O ($ $$ $s   J ?J%>)J2%J/2J;c                    g d}g d}dgt        |      z  }t        | |||      }t        |      dk(  sJ |j                         5 }|d   j                  ddd	}|j	                  |d
d |       d d d        t        |      dk(  sJ |d   j
                  j                  dk(  sJ |d   j
                  j                  dk(  sJ |d   j
                  j                  d
k(  sJ |d
   j
                  j                  dk(  sJ |d   j
                  j                  dv sJ |d   j
                  j                  dk(  sJ y # 1 sw Y   xY w)N)Ifoundapilatesclassnearworkr`   )r2   r2   r   r   r2   r   r   r2   rQ   rR      r   zpilates classOr%   r.   r   r   r   r   r2   r3   )r2   r.   )r   r   r   r   r   r'   r&   )rn   rJ   rS   rT   r    r!   r   s          r"   %test_doc_retokenize_spans_merge_headsrz      s8   HE$E7SZD
he5t
<Cs8q==		 1[Ao3O#a(%01 s8q==q6;;==Aq6;;==Aq6;;==Aq6;;==Aq6;;==F"""q6;;==A1 1s   *D==Ec           	      &   d} | |      }t        j                  t              5  |j                         5 }|j	                  |dd dddd       |j	                  |dd	 dddd       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)
NrO   r   r3   rW   rX   rY   r%   r   r2   )pytestraises
ValueErrorr   r   r4   s       r"   ,test_doc_retokenize_spans_merge_non_disjointr      s    D
t
C	z	" 	^^ 	Aa#mO   Aa#mO  		 		 		 	s"   B5A;*B;B	 BBc                 ^   d}g d}dgt        |      z  } | |      }t        |j                  |D cg c]  }|j                   c}||      }|d   j                  j
                  dk(  sJ |j                         5 }ddd	d
}|j                  |dd |       d d d        |d   j                  j
                  dk(  sJ d}g d}dgt        |      z  } | |      }t        |j                  |D cg c]  }|j                   c}||      }|j                         5 }|j                  D ]9  }	|	j                  |	j                  |	j                  d
}|j                  |	|       ; 	 d d d        d}g d}dgt        |      z  } | |      }t        |j                  |D cg c]  }|j                   c}||      }|j                         5 }|j                  D ]  }	|j                  |	        	 d d d        y c c}w # 1 sw Y   hxY wc c}w # 1 sw Y   xY wc c}w # 1 sw Y   y xY w)N.displaCy is a parse tool built with Javascript)r2   r2   r   r   r2   r   r   r   rQ   rR   r   r2   NPtoolry   r%   r3   r   r   zmdisplaCy is a lightweight and modern dependency parse tree visualization tool built with CSS3 and JavaScript.)r2   r2   
   r   r.   r.   r   r   r   r   r2   r            r   r2   zFOne test with entities like New York City so the ents list is not void)r2   r2   r2   r3   r.   r   r   r   r   r   r   r   r2   r   r   )r   r   r;   r   r'   r&   r   r   entslabel_r   )
r   r   rS   rT   rZ   r[   r    r!   r   ents
             r"   "test_doc_retokenize_span_np_mergesr      s	   ;D$E7SZD$F
fll6":a166":%d
SCq6;;==A		 1[v3?#a(%01 q6;;==AzDGE7SZD$F
fll6":a166":%d
SC		 0[88 	0CJJTEc/	00
 TD?E7SZD$F
fll6":a166":%d
SC		 #[88 	#Cc"	## #- #;1 1 #;0 0 #;# #s6   G;
=H 1H
A	H!H
#H# H
H#H,c                 8   d}g d}dgt        |      z  }g d}g d}dgt        |      z  }d|d<   d	|d
<   d|d<   d|d<   d	|d<    | |      }t        |j                  |D cg c]  }|j                   c}||||      }t        |      dk(  sJ |j	                         5 }	|j
                  D ]@  }
t        d |
D              }|
j                  j                  |d}|	j                  |
|       B 	 d d d        t        |      dk(  sJ y c c}w # 1 sw Y   xY w)NzPStewart Lee is a stand up comedian who lives in England and loves Joe Pasquale.
)r2   r3   r3   r   r   r   r3   rx   r   rx   r   rx   rx      r   r3      rQ   )rW   rW   rc   ra   VBRPrb   WPrc   INrW   CCrc   rW   rW   r`   SP))PERSONr   r3   )rY   r   r   )r   r   r   ry   zB-PERSONr   zI-PERSONr2   zB-GPEr   r   r   )rJ   rS   rT   rh   r      c              3   4   K   | ]  }|j                     y wrA   )r=   )rD   ws     r"   rF   z9test_doc_retokenize_spans_entity_merge.<locals>.<genexpr>   s     411;;4rG   r1   r   r   )
r   r   r;   r   r   r   maxrootr   r   )r   r   rS   rT   rh   r   rZ   r[   r    r!   r   r   r   s                r"   &test_doc_retokenize_spans_entity_merger      s.   ^DBE7SZDxDBD53u:DDGDGDHDHDH$F
%&!qvv&C s8r>>		 0[88 	0C444H!hhoo8DEc/	00 s8r>> '0 0s   "D
!ADDc                     g d}t        t               |      }|j                  j                  j	                  d      ddf|j                  j                  j	                  d      ddfg|_        |d   j                  dk(  sJ |d	   j                  d
k(  sJ |d   j                  d
k(  sJ |d   j                  dk(  sJ |j                         5 }|j                  |dd        d d d        t        |      t        |      d	z
  k(  sJ |d   j                  dk(  sJ |d	   j                  d
k(  sJ g d}t        t               |      }|j                         5 }dd	d}|j                  |dd |       |j                  |dd |       d d d        |d   j                  dk(  sJ |d	   j                  d
k(  sJ g d}t        t               |      }|j                  j                  j	                  d      ddf|j                  j                  j	                  d      ddfg|_        |d   j                  dk(  sJ |d   j                  d
k(  sJ |d   j                  dk(  sJ |d   j                  d
k(  sJ |j                         5 }|j                  |dd        |j                  |dd        |j                  |dd        d d d        t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ g d}g d}dgt        |      z  }d|d<   d|d<   d|d<   d|d<   dgt        |      z  }| j                  j	                  d       | j                  j	                  d       | j                  j	                  d       t        | ||||      }|dd j                  |d   k(  sJ |dd j                  |d   k(  sJ |j                         5 }|j                  |dd        |j                  |dd        |j                  |dd        d d d        t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  d
k(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ g d}g d}dgt        |      z  }d|d<   d|d<   d|d<   d|d<   dgt        |      z  }t        | ||||      }|j                         5 }|j                  |dd        |j                  |dd        d d d        t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ y # 1 sw Y   xY w# 1 sw Y   DxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)N)rs   bcderI   zent-abcr   r.   zent-dr   Br2   rq   r3   )r   ent_iobr   r   )	rs   r   r   r   r   fghr&   zent-dezent-fgr   r   r   )	r   r   r.   r   r   r   r   r   r   ry   zB-ent-dezI-ent-dezB-ent-fgzI-ent-fgrQ   )rJ   rS   rT   r   )	r   r   r.   r   r   r   r   r   r   )r   r   r;   r<   addr   ent_iob_r   r   r   r=   r   )rn   rJ   r    r!   r   rS   r   rT   s           r"   *test_doc_retokenize_spans_entity_merge_iobr      s;   %E
egU
#C					y	)1a0					w	'A.CH q6??c!!!q6??c!!!q6??c!!!q6??c!!!		 $[#a(#$s8s5zA~%%%q6??c!!!q6??c!!! &E
egU
#C		 1[&15#a(%0#a(%01 q6??c!!!q6??c!!! :E
egU
#C					x	(!Q/					x	(!Q/CH q6??c!!!q6??c!!!q6??c!!!q6??c!!!		 $[#a(##a(##a(#$ s8q==q6??c!!!q6x'''q6??c!!!q6x''' :E'E53u:DDGDGDGDG7SZD""
he5t$
GCq8==CF"""q8==CF"""		 $[#a(##a(##a(#$ s8q==q6??c!!!q6x'''q6??c!!!q6x'''q6??c!!!q6x''' :E'E53u:DDGDGDGDG7SZD
he5t$
GC		 $[#a(##a(#$ s8q==q6??c!!!q6x'''q6??c!!!q6x'''e$ $1 1&$ $2$ $,$ $s<   W 2W	=W=W'.)W4 W
WW$'W14W=c                    d}g d}g d} | |      }t        |j                  |D cg c]  }|j                   c}||      }t        |j                        \  }}t        |      }	t        |      }
|j                         5 }ddd}|j                  |dd |	       |j                  |d
d  |	       d d d        t        |j                        \  }}t        |      |	dz
  k(  sJ t        |      |
dz
  k(  sJ y c c}w # 1 sw Y   MxY w)NzOStewart Lee is a stand up comedian. He lives in England and loves Joe Pasquale.)r2   r3   r3   r   r3   r   r   r3   r   r   r   r   r   r   r   r   r   )compoundnsubjROOTdetamodprtattrpunctr   r   preppobjccconjr   dobjr   rR   noner1   r   r3   r   r2   )r   r;   r   r(   sentsr   r   r   )r   r   rS   rT   rZ   r[   r    sent1sent2init_len	init_len2r!   r   s                r"   5test_doc_retokenize_spans_sentence_update_after_merger   [  s   \DBE)D $F
fll6":a166":%d
SC		?LE55zHE
I		 1[ f5#a(%0#bc(%01 		?LE5u:A%%%u:Q&&& #;1 1s   C;
2D  D	c                    d}g d}g d} | |      }t        |j                  |D cg c]  }|j                   c}||      }t        |j                        d   }t        t        |j                  j                              }|j                         5 }	ddd}
|	j                  |dd |
	       d d d        t        t        |j                  j                              |d
z
  k(  sJ y c c}w # 1 sw Y   ?xY w)NzNStewart Lee is a stand up comedian who lives in England and loves Joe Pasquale)r2   r3   r3   r   r   r   r3   rx   r   rx   r   rx   rx   r   r   )r   r   r   r   r   r   r   r   relclr   r   r   r   r   r   rR   r   r   r1   r3   r   r2   )
r   r;   r   r(   r   r   r   subtreer   r   )r   r   rS   rT   rZ   r[   r    r   r   r!   r   s              r"   ,test_doc_retokenize_spans_subtree_size_checkr   q  s    [D;ED $F
fll6":a166":%d
SCOAE4

**+,H		 1[ f5#a(%01 tEJJ&&'(HqL888 #;1 1s   C)
C..C7c                 d   t        j                  ddd       t        j                  ddd       t        | g d      }|j                         5 }d	dd
dd}|j	                  |dd |       d d d        |d   j
                  d	k(  sJ |d   j                  j                  du sJ |d   j                  j                  d
k(  sJ t        | g d      }|j                         5 }|j	                  |dd ddd
di       |j	                  |dd dd ddi       d d d        |d   j                  j                  du sJ |d   j                  j                  d
k(  sJ |d   j                  j                  J |d   j                  j                  dk(  sJ y # 1 sw Y   ,xY w# 1 sw Y   xY w)Nrs   FT)defaultforcer   nothinghelloworld!rI   hello world1)rs   r   )r   _r   r3   r   )r   r   r   r   r   r   2r2   )	r   set_extensionr   r   r   r   r   rs   r   )rn   r    r!   r   s       r"   )test_doc_retokenize_merge_extension_attrsr     s   	U$7	Yd;
h5
6C		 1['Ds.CD#a(%01 q6==M)))q688::q688::
h:
;C		 H[#a(3d0E*FG#a(3d0E*FGH q688::q688::q688::q688::1 1H Hs   F%7F&F#&F/underscore_attrsrs   xr   r   r2   c                 `   t        j                  dd d       t        j                  dd d       t        | g d	      }d
|i}t        j                  t
              5  |j                         5 }|j                  |dd |       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nrs   c                     | S rA    r   s    r"   <lambda>zCtest_doc_retokenize_merge_extension_attrs_invalid.<locals>.<lambda>      a     T)getterr   r   c                     | S rA   r   r   s    r"   r   zCtest_doc_retokenize_merge_extension_attrs_invalid.<locals>.<lambda>  r   r   )methodr   r   rI   r   r   r3   r   )r   r   r   r|   r}   r~   r   r   )rn   r   r    r   r!   s        r"   1test_doc_retokenize_merge_extension_attrs_invalidr     s    	Kt<	Kt<
h5
6C"#E	z	" 5^^ 	5c!Ahe4	55 5	5 	55 5s$   B$0BB$B!	B$$B-c                    t        | g d      }t        d |D              rJ |j                         5 }|j                  |dd ddd	       d
d
d
       |d   j                  dk(  sJ |d   j
                  sJ t        | g d      }t        d |D              rJ t        d |D              rJ |j                         5 }|j                  |dd ddi	       |j                  |dd ddi	       d
d
d
       |d   j                  sJ |d   j
                  sJ |d   j
                  rJ |d   j                  rJ t        | g d      }|d   j                  dk(  sJ |j                         5 }|j                  |dd ddi	       d
d
d
       |d   j                  dk(  sJ | d   j                  dk(  sJ y
# 1 sw Y   pxY w# 1 sw Y   xY w# 1 sw Y   KxY w)a  Test that retokenization also sets attributes on the lexeme if they're
    lexical attributes. For example, if a user sets IS_STOP, it should mean that
    "all tokens with that lexeme" are marked as a stop word, so the ambiguity
    here is acceptable. Also see #2390.
    r   rI   c              3   4   K   | ]  }|j                     y wrA   rB   rD   r[   s     r"   rF   z7test_doc_retokenizer_merge_lex_attrs.<locals>.<genexpr>       *199*rG   r   r3   r   T)r   rC   r   N)einszweir   r   c              3   4   K   | ]  }|j                     y wrA   )like_numr   s     r"   rF   z7test_doc_retokenizer_merge_lex_attrs.<locals>.<genexpr>  s     +!1::+rG   c              3   4   K   | ]  }|j                     y wrA   rB   r   s     r"   rF   z7test_doc_retokenizer_merge_lex_attrs.<locals>.<genexpr>  r   rG   r   r   rC   r2   r   normr   )r   rK   r   r   r   rC   r   norm_rn   r    r!   s      r"   $test_doc_retokenizer_merge_lex_attrsr     s    h5
6C*c****		 U[#a(Md*STUq6==M)))q6>>>
h8
9C+s++++*c****		 =[#a(:t*<=#a(9d*;<= q6???q6>>>1v~~1v
h8
9Cq6<<6!!!		 9[#a(63-89q6<<3F!!V++++U U= =9 9s#   F<1G	2G<G	GGc                    t        | g d      }|j                         5 }|j                  |dd        |j                  |dd        ddd       t        |      dk(  sJ |d   j                  dk(  sJ y# 1 sw Y   .xY w)zsTest that the retokenizer automatically skips duplicate spans instead
    of complaining about overlaps. See #3687.r   rI   r   r3   Nr   )r   r   r   r   r   r   s      r"   test_retokenize_skip_duplicatesr     s     h5
6C		 $[#a(##a(#$ s8q==q6;;-'''	$ $s   )A66A?c                     t        | g d      }t        j                  t              5  |j	                         5 }|j                  |dd        d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   rI   r2   )r   r|   r}   r~   r   r   r   s      r"   $test_retokenize_disallow_zero_lengthr     sj    
h5
6C	z	" (^^ 	(c!Ah'	(( (	( 	(( (s"   A,A A, A)	%A,,A5c                 >   d}g d} | |      }t        |j                  |D cg c]  }|j                   c}|      }t        t	        |j
                              dk(  sJ |j                         5 }|j                  |dd        d d d        t        t	        |j
                              dk(  sJ t        |j                  |D cg c]  }|j                   c}|      }t        t	        |j
                              dk(  sJ |j                         5 }|j                  |dd        d d d        |d   j                  J t        |j                  |D cg c]  }|j                   c}|      }t        t	        |j
                              dk(  sJ |j                         5 }|j                  |dd dd	i
       d d d        t        t	        |j
                              dk(  sJ y c c}w # 1 sw Y   kxY wc c}w # 1 sw Y   xY wc c}w # 1 sw Y   UxY w)Nr   )r2   r   r   r   r2   r   r   r   )rJ   sent_startsr3   r2   r.   r   
sent_startTr   )	r   r;   r   r   r(   r   r   r   is_sent_start)r   r   r   rZ   r[   r    r!   s          r"   3test_doc_retokenize_merge_without_parse_keeps_sentsr     s   ;D*K$F fll6":a166":
TCtCII1$$$		 $[#a(#$tCII1$$$ fll6":a166":
TCtCII1$$$		 $[#a(#$q6''' fll6":a166":
TCtCII1$$$		 @[#a(<*>?@tCII1$$$% #;$ $
 #;$ $
 #;@ @s5   G+
2G0G=
HH
&H0G:HH)r|   spacy.attrsr   spacy.tokensr   r   spacy.vocabr   r#   r,   r5   r7   r>   rM   r\   ro   rz   r   r   r   r   r   r   r   markparametrizer   r   r   r   r   r   r   r"   <module>r      s      # $42L+	%
%$)&X$#@B^(B',9$, +sCj3*sCjSTRU-VW5 X5,B((%r   