
    i\?                        d dl Z d dlmZmZ d dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ e j                   ej                  d      d	               Ze j                  d
        Ze j                  d        Ze j                  d        Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z e jB                  jE                  dddg      d        Z#d Z$d Z%d  Z&d! Z'd" Z(d# Z)d$ Z*d% Z+d& Z,d' Z-d( Z.y))    N)NumpyOpsget_current_ops)registry)MatchPatternError)make_tempdir)Span)Examplespan_ruler_patternsc                  T    dddddddiddigddd	digd
dddddgdddddddgS )NHELLOhello worldhello1)labelpatternidBYELOWERbyer   r   ORTHhello2COMPLEXfoo*)r   OPTECH_ORGApple	Microsoft r       u/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/spacy/tests/pipeline/test_span_ruler.pypatternsr"      sa     m8Dgu%57G$HI'8&9J%s)C(DE15 r    c                      ddddddgS )NFOOBARzfoo barr   BARBAZzbar bazr   r   r    r!   overlapping_patternsr&      s     y1y1 r    c                      dddddddddgS )NPERSONDinar   ORGACMEACMr   r   r    r!   person_org_patternsr-   !   s&     v.F+E* r    c                     | dddgz   S )NDATE	June 14thr   r   )r-   s    r!   person_org_date_patternsr1   *   s    F{"K!LLLr    c                    t        j                  d      }|j                  dddi      }|j                  |        t	        d |j
                  j                  j                         D              }|dkD  sJ |j                  g        t	        d |j
                  j                  j                         D              }||k(  sJ y	)
z/Test that patterns don't get added excessively.xx
span_rulervalidateTconfigc              3   2   K   | ]  }t        |        y wNlen.0mms     r!   	<genexpr>z,test_span_ruler_add_empty.<locals>.<genexpr>4   s     KBBK   r   c              3   2   K   | ]  }t        |        y wr9   r:   r<   s     r!   r?   z,test_span_ruler_add_empty.<locals>.<genexpr>7   s     I"c"gIr@   N)spacyblankadd_pipeadd_patternssummatcher	_patternsvalues)r"   nlprulerpattern_countafter_counts        r!   test_span_ruler_add_emptyrN   /   s    
++d
CLLz4.@LAE	x K%--*A*A*H*H*JKKM1	rI(?(?(F(F(HIIK-'''r    c                 B   t        j                  d      }|j                  d      }|j                  |        t	        |      t	        |       k(  sJ t	        |j
                        dk(  sJ d|v sJ d|v sJ  |d      }t	        |j                  d         dk(  sJ |j                  d   d	   j                  dk(  sJ |j                  d   d	   j                  d
k(  sJ |j                  d   d   j                  dk(  sJ |j                  d   d   j                  dk(  sJ y )Nr3   r4      r   r   hello world bye byerK      r   r       )	rB   rC   rD   rE   r;   labelsspanslabel_id_r"   rJ   rK   docs       r!   test_span_ruler_initr[   ;   s"   
++d
CLL&E	x u:X&&&u||!!!eE>>
#
$Csyy!"a'''99Wa ''722299Wa $$00099Wa ''500099Wa $$***r    c                  n   t        j                  d      } | j                  d      }t        |      dk(  sJ t        |j                        dk(  sJ | j
                  dgk(  sJ t        j                  t              5   | d      }d d d        t        j                  d         dk(  sJ y # 1 sw Y   'xY w)Nr3   r4   r   rQ   rK   )
rB   rC   rD   r;   rU   
pipe_namespytestwarnsUserWarningrV   )rJ   rK   rZ   s      r!   !test_span_ruler_no_patterns_warnsra   K   s    
++d
CLL&Eu:??u||!!!>>l^+++	k	" )'()syy!"a''') )s   <	B++B4c                    t        j                  d      }|j                  d      }t        |j                        dk(  sJ |j                  d |        t        |j                        dk(  sJ  |d      }|j                  d   d   j                  d	k(  sJ |j                  d   d
   j                  dk(  sJ |j                  d       dddii|j                  d   d   d<   |j                  d      }t        |j                        dk(  sJ |j                          t        |j                        dk(  sJ  |d      }|j                  d   d   j                  d	k(  sJ |j                  d   d
   j                  dk(  sJ y )Nr3   r4   r   c                      g S r9   r   r   r    r!   <lambda>z/test_span_ruler_init_patterns.<locals>.<lambda>[       R r    )r"   rP   rQ   rK   r   rS   r   r"   @miscr
   
initialize
components)
rB   rC   rD   r;   rU   rg   rV   rW   remove_piper7   rY   s       r!   test_span_ruler_init_patternsrj   V   sx   
++d
CLL&Eu||!!!	Z(3u||!!!
#
$C99Wa ''722299Wa ''5000OOL! 	W34<CJJ|\*<8 LL&Eu||!!!NNu||!!!
#
$C99Wa ''722299Wa ''5000r    c                     t        j                  d      }|j                  d      }|j                  |        t	        |j
                        dk(  sJ |j                  d        t	        |j
                        dk(  sJ y)z)Test that initialization clears patterns.r3   r4   rP   c                      g S r9   r   r   r    r!   rd   z,test_span_ruler_init_clear.<locals>.<lambda>t   re   r    r   N)rB   rC   rD   rE   r;   rU   rg   )r"   rJ   rK   s      r!   test_span_ruler_init_clearrm   n   sh    
++d
CLL&E	x u||!!!	Z u||!!!r    c                    t        j                  d      }|j                  d      }|j                  |        t	        |j
                        dk(  sJ  |d      }t	        |j                  d         dk(  sJ |j                          t	        |j
                        dk(  sJ t        j                  t              5   |d      }d d d        t	        |j                  d         dk(  sJ y # 1 sw Y   'xY w)Nr3   r4   rP   r   rK   rS   r   )rB   rC   rD   rE   r;   rU   rV   clearr^   r_   r`   rY   s       r!   test_span_ruler_clearrp   x   s    
++d
CLL&E	x u||!!!
m
Csyy!"a'''	KKMu||!!!	k	" !- !syy!"a'''! !s   :	C))C2c                 H   t        j                  d      }|j                  dddi      }|j                  |        |j	                  d      }|dd g|j
                  d	<    ||      }t        |j
                  d	         d
k(  sJ |j
                  d	   d   |dd k(  sJ |j
                  d	   d   j                  dk(  sJ |j
                  d	   d   j                  dk(  sJ |j
                  d	   d   j                  dk(  sJ |j
                  d	   d   j                  dk(  sJ y )Nr3   r4   	overwriteFr6   OH HELLO WORLD bye byer   rR   rK      rS   r   r   r   rT   )	rB   rC   rD   rE   make_docrV   r;   rW   rX   rY   s       r!   test_span_ruler_existingrv      s)   
++d
CLL{E.BLCE	x 
,,/
0Ca(CIIg
c(Csyy!"a'''99Wa C!H,,,99Wa ''722299Wa $$00099Wa ''500099Wa $$***r    c                    t        j                  d      }|j                  dddi      }|j                  |        |j	                  d      }|dd g|j
                  d	<    ||      }t        |j
                  d	         dk(  sJ |j
                  d	   d   j                  d
k(  sJ |j
                  d	   d   j                  d
k(  sJ |j
                  d	   d   j                  dk(  sJ y )Nr3   r4   rr   Tr6   rs   r   rR   rK   r   rS   r   )	rB   rC   rD   rE   ru   rV   r;   rW   textrY   s       r!   "test_span_ruler_existing_overwritery      s    
++d
CLL{D.ALBE	x 
,,/
0Ca(CIIg
c(Csyy!"a'''99Wa ''722299Wa %%00099Wa ''5000r    c                    t        j                  d      }|j                  d      }|j                  |        t	        |      t	        |       k(  sJ t	        |j
                        dk(  sJ |j                         }t        j                  d      }|j                  d      }t	        |      dk(  sJ t	        |j
                        dk(  sJ |j                  |      }t	        |      t	        |       k(  sJ t	        |j
                        dk(  sJ t	        |j                        t	        |j                        k(  sJ |j                  D ]  }||j                  v rJ  t        |j
                        t        |j
                        k(  sJ y )Nr3   r4   rP   r   )
rB   rC   rD   rE   r;   rU   to_bytes
from_bytesr"   sorted)r"   rJ   rK   ruler_bytesnew_nlp	new_rulerr   s          r!   test_span_ruler_serialize_bytesr      sX   
++d
CLL&E	x u:X&&&u||!!!.."Kkk$G  .Iy>Qy A%%%$$[1Iy>S]***y A%%%y!!"c%..&9999>> -),,,,,-)""#vell';;;;r    c                     t        j                  d      } | j                  d      }| j                  ddddi      }dddigd	}dd
digd	}t        j                  t
              5  |j                  |g       d d d        |j                  |g       t        j                  t              5  |j                  |g       d d d        y # 1 sw Y   PxY w# 1 sw Y   y xY w)Nr3   r4   validated_span_rulerr5   T)namer7   r   r   r   ASDF)rB   rC   rD   r^   raises
ValueErrorrE   r   )rJ   rK   validated_rulervalid_patterninvalid_patterns        r!   test_span_ruler_validater      s    
++d
CLL&Ell1:t:L # O &GW3E2FGM 'fg5F4GHO 
z	" .O,-.   -1 
(	) 8$$o%678 8. .8 8s   &C,CCCc           
          t        j                  d      }|j                  dddi      }|j                  |        t	        |j
                        t	        t        | D cg c]  }|d   	 c}            k(  sJ y c c}w )Nr3   r4   rr   Tr6   r   )rB   rC   rD   rE   r}   rU   set)r"   rJ   rK   ps       r!   test_span_ruler_propertiesr      sh    
++d
CLL{D.ALBE	x %,,6#8.Laqz.L*M#NNNN.Ls   A:c                 ^   t        j                  d      }|j                  d      }|j                  |         ||j	                  d            }t        |j                  d         dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d   d   j                  d	k(  sJ y )
Nr3   r4   foo bar bazrK   rR   r   r$   rS   r%   rB   rC   rD   rE   ru   r;   rV   rW   r&   rJ   rK   rZ   s       r!   !test_span_ruler_overlapping_spansr      s    
++d
CLL&E	+,
]+
,Csyy!"a'''99Wa ''833399Wa ''8333r    c                    t        j                  d      }|j                  d      }|j                  |        d} ||j	                  |            }t        |j                  d         dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d   d   j                  d	k(  sJ |j	                  |      }t        |ddd
      g|j                  d<   |j                  t        ||      g      }|d   dk(  sJ |d   dk(  sJ y )Nr3   r4   r   rK   rR   r   r$   rS   r%   r   spans_ruler_pg      ?spans_ruler_rg      ?)rB   rC   rD   rE   ru   r;   rV   rW   r   evaluater	   )r&   rJ   rK   rx   pred_docref_docscoress          r!   test_span_ruler_scorerr      s   
++d
CLL&E	+,DS\\$'(Hx~~g&'1,,,>>'"1%,,888>>'"1%,,888ll4 G"7AqABGMM'\\78W567F/"c)))/"c)))r    	n_processrS   rR   c                 2   t        t        t              s| dk  r~dg}dddg}t        j                  d      }|j                  d      }|j                  |       |j                  |d      D ]'  }|j                  d	   D ]  }|j                  dk(  rJ  ) y y )
NrR   zI enjoy eating Pizza Hut pizza.FASTFOODz	Pizza Hutr   r3   r4   )r   rK   )

isinstancer   r   rB   rC   rD   rE   piperV   rW   )r   textsr"   rJ   rK   rZ   ents          r!   test_span_ruler_multiprocessingr      s    /8,	A23([ABkk$\*8$88EQ8/ 	0Cyy) 0zzZ///0	0 1>r    c                    t        j                  d      }|j                  d      }|j                  |        t	               5 }|j                  |dz         |j                  |dz         t        j                  t              5  |j                  |dz         d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr3   r4   
test_rulernon_existing_dir)
rB   rC   rD   rE   r   to_disk	from_diskr^   r   r   )r"   rJ   rK   ds       r!   test_span_ruler_serialize_dirr      s    
++d
CLL&E	x 	 41a,&'L()]]:& 	4OOA 223	44 4	4 	44 4s%   AB6B*B6*B3	/B66B?c                 P   t        j                  d      }|j                  d      }|j                  |         ||j	                  d            }t        |j                        dk(  sJ t        |j                  d         dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d   d   j                  d	k(  sJ |j                  d        ||j	                  d            }t        |j                  d         dk(  sJ t        |j                        d
k(  sJ y )Nr3   r4   Dina went to schoolrt   rK   rS   r   r(   r)   rR   rB   rC   rD   rE   ru   r;   r"   rV   rW   rx   remover-   rJ   rK   rZ   s       r!   test_span_ruler_remove_basicr   	  s   
++d
CLL&E	*+
23
4Cu~~!###syy!"a'''99Wa ''833399Wa %%///	LL
23
4Csyy!"a'''u~~!###r    c                    t        j                  d      }|j                  d      }|j                  |        t	        |j
                        dk(  sJ t        j                  t              5  |j                  d       d d d        t        j                  t              5  |j                  d       d d d        y # 1 sw Y   =xY w# 1 sw Y   y xY w)Nr3   r4   rt   NE)rB   rC   rD   rE   r;   r"   r^   r   r   r   remove_by_id)r-   rJ   rK   s      r!   *test_span_ruler_remove_nonexisting_patternr     s    
++d
CLL&E	*+u~~!###	z	" T	z	" !4 ! ! ! !s   +B9C9CCc                 t   t        j                  d      }|j                  d      }|j                  |         ||j	                  d            }t        |j                        dk(  sJ t        |j                  d         dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d   d   j                  d	k(  sJ |j                  d   d
   j                  dk(  sJ |j                  d   d
   j                  dk(  sJ |j                  d        ||j	                  d            }t        |j                        dk(  sJ t        |j                  d         d
k(  sJ |j                  d   d   j                  dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d       t        j                  t              5   ||j	                  d            }t        |j                        dk(  sJ t        |j                  d         dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr3   r4   zDina founded the company ACME.rt   rK   rR   r   r(   r)   rS   r*   r+   zDina founded the company ACME)rB   rC   rD   rE   ru   r;   r"   rV   rW   rx   r   r^   r_   r`   r   s       r!   'test_span_ruler_remove_several_patternsr   #  s   
++d
CLL&E	*+
=>
?Cu~~!###syy!"a'''99Wa ''833399Wa %%///99Wa ''500099Wa %%///	LL
<=
>Cu~~!###syy!"a'''99Wa ''500099Wa %%///	LL	k	" ,CLL!@AB5>>"a'''399W%&!+++, , ,s   AH..H7c                    t        j                  d      }|j                  d      }|j                  |         ||j	                  d            }t        |j                  d         dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d   d	   j                  d
k(  sJ |j                  d   d	   j                  dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d   d   j                  dk(  sJ |j                  d
       |j                  d        ||j	                  d            }t        |j                  d         d	k(  sJ y )Nr3   r4   *Dina founded the company ACME on June 14thrK   rt   r   r(   r)   rS   r*   r+   rR   r/   r0   r   )
rB   rC   rD   rE   ru   r;   rV   rW   rx   r   r1   rJ   rK   rZ   s       r!   (test_span_ruler_remove_patterns_in_a_rowr   ;  sq   
++d
CLL&E	/0
IJ
KCsyy!"a'''99Wa ''833399Wa %%///99Wa ''500099Wa %%///99Wa ''611199Wa %%444	LL	LL
23
4Csyy!"a'''r    c                 n   t        j                  d      }|j                  d      }|j                  |        t	        |j
                        dk(  sJ |j                  d       t	        |j
                        dk(  sJ |j                  d       t	        |j
                        dk(  sJ |j                  d       t	        |j
                        d	k(  sJ t        j                  t              5   ||j                  d
            }t	        |j                  d         d	k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr3   r4   rP   r(   rt   r*   rS   r/   r   r   rK   )rB   rC   rD   rE   r;   r"   r   r^   r_   r`   ru   rV   r   s       r!   #test_span_ruler_remove_all_patternsr   M  s    
++d
CLL&E	/0u~~!###	LLu~~!###	LLu~~!###	LLu~~!###	k	" ,CLL!MNO399W%&!+++, , ,s   ,5D++D4c                     t        j                  d      } | j                  d      }dddg}|j                  |        || j	                  d            }t        |j                        dk(  sJ t        |j                  d         dk(  sJ |j                  d   d	   j                  dk(  sJ |j                  d   d	   j                  dk(  sJ d
ddg}|j                  |        || j	                  d            }t        |j                        dk(  sJ t        |j                  d         dk(  sJ |j                  d   d	   j                  dk(  sJ |j                  d   d	   j                  dk(  sJ |j                  d   d   j                  d
k(  sJ |j                  d   d   j                  dk(  sJ |j                  d        || j	                  d            }t        |j                        dk(  sJ t        |j                  d         dk(  sJ |j                  d   d	   j                  d
k(  sJ |j                  d   d	   j                  dk(  sJ |j                  |        || j	                  d            }t        |j                        dk(  sJ t        |j                  d         dk(  sJ dddg}|j                  |        || j	                  d            }t        |j                        dk(  sJ t        |j                  d         dk(  sJ |j                  d        || j	                  d            }t        |j                        dk(  sJ t        |j                  d         dk(  sJ y )Nr3   r4   DATE1z	last timer   z=I saw him last time we met, this time he brought some flowersrS   rK   r   DATE2z	this timerR   DATE3zanother timez[I saw him last time we met, this time he brought some flowers, another time some chocolate.rt   r   )rJ   rK   	patterns1rZ   	patterns2	patterns3s         r!   test_span_ruler_remove_and_addr   ]  sq   
++d
CLL&E"{;<I	y!
TUC u~~!###syy!"a'''99Wa ''722299Wa %%444"{;<I	y!
TUC u~~!###syy!"a'''99Wa ''722299Wa %%44499Wa ''722299Wa %%444	LL
TUC u~~!###syy!"a'''99Wa ''722299Wa %%444	y!
TUC u~~!###syy!"a'''"~>?I	y!
i	
C
 u~~!###syy!"a'''	LL
i	
C
 u~~!###syy!"a'''r    c                 (   t        j                  d      }|j                  ddddii      }|j                  |         ||j	                  d            }t        |j                  d         d	k(  sJ |j                  d   d
   j                  dk(  sJ y )Nr3   r4   spans_filterrf   z#spacy.first_longest_spans_filter.v1r6   r   rK   rS   r   r$   r   r   s       r!   test_span_ruler_spans_filterr     s    
++d
CLL*O PQ  E 
+,
]+
,Csyy!"a'''99Wa ''8333r    c                    t        j                  d      }|j                  dddi      }|j                  |         ||j	                  d            }t        |j                        dk(  sJ |j                  d   j                  d	k(  sJ y )
Nr3   r4   annotate_entsTr6   r   rS   r   r$   )rB   rC   rD   rE   ru   r;   entsrW   r   s       r!   #test_span_ruler_ents_default_filterr     s{    
++d
CLL.ELFE	+,
]+
,Csxx=A88A;)))r    c                    t        j                  d      }|j                  dddddid      }|j                  |        |j	                  d	      }t        |d
dd      t        |ddd      g|_         ||      }t        |j                        dk(  sJ |j                  d   j                  dk(  sJ |j                  d
   j                  dk(  sJ y )Nr3   r4   TFrf   z#spacy.prioritize_new_ents_filter.v1)r   rr   ents_filterr6   zfoo bar baz a b crS   rt   r%   r      ABCrR   r   r$   )	rB   rC   rD   rE   ru   r   r   r;   rW   r   s       r!   %test_span_ruler_ents_overwrite_filterr     s    
++d
CLL!#%JK
  E 
+,
,,*
+CS!Qh/c1au1MNCH
*Csxx=A88A;)))88A;&&&r    c                 @   t        j                  d      d        }t        j                  d      }|j	                  ddddid      }|j                  |        t        j                  t              5   ||j                  d	             d d d        y # 1 sw Y   y xY w)
Ntest_pass_through_filterc                      d } | S )Nc                     | |z   S r9   r   )spans1spans2s     r!   pass_through_filterz^test_span_ruler_ents_bad_filter.<locals>.make_pass_through_filter.<locals>.pass_through_filter  s    F?"r    r   )r   s    r!   make_pass_through_filterzAtest_span_ruler_ents_bad_filter.<locals>.make_pass_through_filter  s    	# #"r    r3   r4   Trf   )r   r   r6   r   )
r   miscrB   rC   rD   rE   r^   r   r   ru   )r&   r   rJ   rK   s       r!   test_span_ruler_ents_bad_filterr     s    ]]-.# /# ++d
CLL!#%?@
  E 
+,	z	" +cll=)*+ + +s   3BB)/r^   	thinc.apir   r   rB   r   spacy.errorsr   spacy.tests.utilr   spacy.tokensr   spacy.trainingr	   fixturer   r"   r&   r-   r1   rN   r[   ra   rj   rm   rp   rv   ry   r   r   r   r   r   markparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   <module>r      s6    /   * )  " $% &      M M	(+ (10"(+
1<(8,O4*" q!f-0 .04$!,0($, 5(p	4*'(+r    