
    iN                     T   d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ ddlmZmZ dg dg ddfdg dg ddfgZdg dg ddfdg dg ddfgZdg dg ddfgZdZe j>                  d        Z e j>                  d        Z!d Z"e jF                  jI                  d      d         Z%e jF                  jI                  d!      d"        Z&e jF                  jI                  d!      d#        Z'e jF                  jI                  d$      e jF                  jQ                  d%&      d'               Z)d( Z*e jF                  jW                  d)&      e jF                  jY                  d*d+gg      d,               Z-e jF                  jW                  d)&      d-        Z.d. Z/d/ Z0e jF                  jW                  d)&      d0        Z1d1 Z2d2 Z3e jF                  jY                  d3d4d5g      d6        Z4e jF                  jY                  d3d4d5g      d7        Z5e jF                  jl                  e jF                  jY                  d3d4d5g      e jF                  jY                  d8d9ed4d:d;dd<d=d>ed4d:d;dd<d=g      d?                      Z7d@ Z8dA Z9y)B    N)assert_equal)Adam)registryutil)DEPNORM)English)DependencyParser)DEFAULT_PARSER_MODEL)DEFAULT_TOK2VEC_MODEL)Doc)Example)Vocab   )apply_transition_sequencemake_tempdirz&They trade mortgage-backed securities.)   r      r      r   r   )nsubjROOTcompoundpunctnmoddobjr   headsdepszI like London and Berlin.)r   r   r   r   r   r   )r   r   r   ccconjr   )r   r   r   r   r   r   )r   r   r   r   r    r   I like London.)r   r   r   N)r   r   r   Ng?c                  (    t        t        d i      S )Nc                     | S N )ss    n/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/spacy/tests/parser/test_parse.py<lambda>zvocab.<locals>.<lambda>C   s    1     )lex_attr_getters)r   r   r%   r)   r'   vocabr+   A   s    4"566r)   c                    | j                   j                  d       dt        i}t        j                  |d      d   }t        | |      dj                  d<   dj                  d<   j                  d	       j                  fd
       t        d      }t        d      D ]E  }i }t        | g d      }t        j                  |g dg dd      }j                  |g||       G S )Nr   modelTvalidater   token_vector_width    hidden_widthleftc                      t               gS r$   _parser_exampleparsers   r'   r(   zparser.<locals>.<lambda>P       v67 r)   gMbP?
   abcdwordsr   r      rC   )r3   r   r3   r   r   sgdlosses)stringsaddr   r   resolver
   cfg	add_label
initializer   ranger   r   	from_dictupdate)	r+   rJ   r-   rE   irF   docexampler8   s	           @r'   r8   r8   F   s    	MMf(
)CS409EeU+F'(FJJ#$!#FJJ~
V
78
u+C2Y 9%34##<1QR
 	wiS89 Mr)   c                 r    t        | j                  g d      }g dg dd}t        j                  ||      S )Nr;   r@   rB   )rightr   r3   r   r   )r   r+   r   rN   )r8   rQ   golds      r'   r6   r6   ]   s1    
fll"6
7C!+LMDS$''r)   i
  c                 v    g d}g d}dgt        |      z  }t        | |||      }|d   j                  du sJ y)zATest that deprojectivization doesn't mess up sentence boundaries.)Whenwewriteorcommunicate	virtually,rX   canhideourtruefeelings.)r   r   	   r   r   r   rd   rd   rd   rd      re   rd   rd   deprA   r   r   r   FN)lenr   is_sent_start)en_vocabrA   r   r   rQ   s        r'   test_issue2772rk   c   sI     @E 9E7SZD
he5t
<Cq65(((r)   i  c                     ddi} t        j                  dt        id      d   }t        t	               |fi | j                  d       dj                  vsJ j                  fd       dj                  vsJ y	)
zBTest that the parser doesn't have subtok label if not learn_tokenslearn_tokensFr-   Tr.   r   subtokc                      t               gS r$   r5   r7   s   r'   r(   z*test_issue3830_no_subtok.<locals>.<lambda>{   r9   r)   Nr   rI   r   r
   r   rK   labelsrL   configr-   r8   s     @r'   test_issue3830_no_subtokrt   q   s     	F g';<tLWUEegu77F
W6==(((
786==(((r)   c                     ddi} t        j                  dt        id      d   }t        t	               |fi | j                  d       dj                  vsJ j                  fd       dj                  v sJ y)	zATest that the parser does have subtok label if learn_tokens=True.rm   Tr-   r.   r   rn   c                      t               gS r$   r5   r7   s   r'   r(   z,test_issue3830_with_subtok.<locals>.<lambda>   r9   r)   Nrp   rr   s     @r'   test_issue3830_with_subtokrw      s     	F g';<tLWUEegu77F
W6==(((
78v}}$$$r)   i$  zNot fixed yet)reasonc                     t        | j                  g d      }d|d   _         | |      }|d   j                  dk(  sJ y )Nr;   r@   Fr   )r   r+   ri   )r8   rQ   s     r'   test_partial_annotationrz      sD     fll"6
7C CF +Cq65(((r)   c                     g d}g d}g d}t        | |||      }|D ]"  }|j                  dk7  rJ |j                          y )N)rP   dozn'thaveother
assistance)rC   rC   rC   rC   r   rC   )r   auxnegr   amodr   rg   r   )r   rf   text)rj   rA   r   r   rQ   ts         r'   test_parser_rootr      sI    =EE:D
he5t
<C "uuz!166!z"r)   z=The step_through API was removed (but should be brought back)rA   Helloc                     t        | |dgdg      }t        |      dk(  sJ |j                  |      5 }	 d d d        |d   j                  dk7  sJ y # 1 sw Y   xY w)Nr   r   rg   r   )r   rh   step_throughrf   )rj   	en_parserrA   rQ   _s        r'   #test_parser_parse_one_word_sentencer      sd    
 heA3fX
>Cs8q==				$ q6::?? s   AAc                 8   g d}g d}t        | |      }t        |||       |d   j                  j                  dk(  sJ |d   j                  j                  dk(  sJ |d   j                  j                  dk(  sJ |d   j                  j                  dk(  sJ y )N)Iatethepizzawith	anchoviesrc   )L-nsubjSzL-detr@   r   r   r   rC   )r   r   headrP   )rj   r   rA   
transitionrQ   s        r'   test_parser_initialr      s     CE*J
he
$Cij9q6;;==Aq6;;==Aq6;;==Aq6;;==Ar)   c                 d   g d}g d}dgt        |      z  }t        | |||      }t        t        |d   j                              dk(  sJ t        t        |d   j                              dk(  sJ t        t        |d   j
                              dk(  sJ t        t        |d   j                              dk(  sJ t        t        |d   j                              d	k(  sJ t        t        |d   j
                              dk(  sJ t        t        |d   j                              d
k(  sJ y )N)Thefourwheelsonr   busturnedquickly)r   r      r   r   rC   r   r   rf   rg   r   r   rC   r   r   r   )rh   r   listleftsrightschildrensubtree)rj   r   rA   r   r   rQ   s         r'   test_parser_parse_subtreesr      s   NE$E7SZD
he5t
<CtCFLL!"a'''tCFMM"#q(((tCFOO$%***tCFLL!"a'''tCFMM"#q(((tCFOO$%***tCFNN#$)))r)   c                    g d}g d}g d}g d}t        | ||||      }|j                         5 }|j                  D ]!  }|j                  |d|j                  i       # 	 d d d        |d   j
                  d	k(  sJ |d
   j
                  dk(  sJ |d   j
                  dk(  sJ |d   j
                  dk(  sJ y # 1 sw Y   ZxY w)N)Aphraser   anotherr   occurs)r   r   r   r   r   r   )detr   prepr   pobjr   )DETNOUNADPr   r   VERB)rA   r   r   poslemma)attrsr   zA phraser   r   r   zanother phraserC   r   )r   
retokenizenoun_chunksmergelemma_r   )rj   rA   r   r   r   rQ   retokenizernps           r'   test_parser_merge_ppr      s    BEE:D
7C
he$e
EC		 >[// 	>Bb"))(<=	>> q6;;*$$$q6;;&   q6;;****q6;;("""> >s   1B<<Cc                 
   g d}g d}t        | |      }t        |||       |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ g d	}t        | |      }t        |||       |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  dk(  sJ |d   j                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ |d   j                  j
                  dk(  sJ y )
N)r<   r=   r>   r?   e)R-nsubjDr   r   r   zR-ROOTr@   r   r   r   r   rC   )r   r   r   r   r   r   r   )r   r   n_leftsn_rights	left_edgerP   
right_edger   )rj   r   rA   r   tokenss        r'   $test_parser_arc_eager_finalize_stater      s    &EFJ'Fi<!9!!!!9"""!9  A%%%!9!!Q&&&!9>>q   !9!!!!9"""!9  A%%%!9!!Q&&&!9>>q   !9!!!!9"""!9  A%%%!9!!Q&&&!9>>q   !9!!!!9"""!9  A%%%!9!!Q&&&!9>>q   !9!!!!9"""!9  A%%%!9!!Q&&&!9>>q    MJ'Fi<!9!!!!9"""!9  A%%%!9!!Q&&&!9>>q   !9!!!!9"""!9  A%%%!9!!Q&&&!9>>q   !9!!!!9"""!9  A%%%!9!!Q&&&!9>>q   !9!!!!9"""!9  A%%%!9!!Q&&&!9>>q   !9!!!!9"""!9  A%%%!9!!Q&&&!9>>q   r)   c                    g d}g d}g d}t        | |||      }t        t        |            D ]3  }|dk(  s|dk(  r||   j                  du rJ ||   j                  du r3J  |j                  D ]  }|D ]  }|j
                  |v rJ   y )	N)&EinSatzrc   u	   AußerdemistZimmerdavonu
   überzeugtr]   dassauchzepige-
netischeMechanismeneineRollespielenr]   alsou	   Vorgänger]   dier   sichdarauf	auswirkenr]   welcheGene	abgelesenwerdenundr   r   nichtrc   r   )&r   r   r      r   r      r   r         r      r   r      r   r   r      r   r         r   r   r   r         r      r   r1   r   r   r   $   )&nkr   r   mor   sboppdr   cpr   r    r   r   r   oarer   r   appr   r   r   r   r   rcr   r   r   ocr   cdr   r   ngr   r   )rA   r   r   r   rC   TF)r   rM   rh   ri   sentsr   )rj   rA   r   r   rQ   rP   senttokens           r'   test_parser_set_sent_startsr   %  s     VE ZE D
he$e
<C3u: 16Q!Vq6''4///q6''5000	1
 		 & 	&E::%%%	&&r)   c                     dddd}dt         i}t        j                  |d      d   }t        | |fi | t        | |       y )NFr   d   )rm   min_action_frequpdate_with_oracle_cut_sizer-   Tr.   )r   r   rI   r
   )rj   rs   rJ   r-   s       r'   test_parser_constructorr  6  sO    '*F
 (
)CS409EXu//Xu%r)   	pipe_namer8   beam_parserc                    t               }|j                  |       }g t        D ]d  \  }}j                  t	        j
                  |j                  |      |             |j                  dg       D ]  }||j                  |        f |j                  fd      }t        d      D ]  }i }|j                  ||        |    dk  sJ d}	 ||	      }
|
d   j                  d	k(  sJ |
d
   j                  dk(  sJ |
d   j                  j                  dk(  sJ |
d
   j                  j                  dk(  sJ y )Nr   c                       S r$   r%   )train_exampless   r'   r(   z&test_incomplete_data.<locals>.<lambda>M  s    N r)   )get_examples   rD   -C6?I like securities.r   r   r   r   r   )r	   add_pipePARTIAL_DATAappendr   rN   make_docgetrK   rL   rM   rO   dep_r   rP   )r  nlpr8   r   annotationsrf   	optimizerrP   rF   	test_textrQ   r  s              @r'   test_incomplete_datar  B  sH    )C\\)$FN) &kg//T0BKPQ??62. 	&C  %	&&
 ,BCI3Z A

>y
@A )v%%% %I
i.Cq6;;'!!!q6;;&   q6;;==Aq6;;==Ar)   c                 0   t               }|j                  |       }g }t        D ]a  \  }}|j                  t	        j
                  |j                  |      |             |j                  dg       D ]  }|j                  |        c |j                         }t        d      D ]  }i }	|j                  |||	        	|    dk  sJ d}
 ||
      }|d   j                  dk(  sJ |d   j                  d	k(  sJ |d
   j                  dk(  sJ |d   j                  j                  dk(  sJ |d   j                  j                  dk(  sJ |d
   j                  j                  dk(  sJ t               5 }|j!                  |       t#        j$                  |      } ||
      }|d   j                  dk(  sJ |d   j                  d	k(  sJ |d
   j                  dk(  sJ |d   j                  j                  dk(  sJ |d   j                  j                  dk(  sJ |d
   j                  j                  dk(  sJ 	 d d d        g d}|j'                  |      D cg c]  }|j)                  t*        g       }}|j'                  |      D cg c]  }|j)                  t*        g       }}|D cg c]
  } ||       c}D cg c]  }|j)                  t*        g       }}t-        ||       t-        ||       y # 1 sw Y   xY wc c}w c c}w c c}w c c}w )Nr      rD   r  r  r   r   r   r   rC   r   r   )zJust a sentence.z$Then one more sentence about London.zHere is another one.r!   )r	   r  
TRAIN_DATAr  r   rN   r  r  rK   rL   rM   rO   r  r   rP   r   to_diskr   load_model_from_pathpipeto_arrayr   r   )r  r  r8   r  r   r  rf   r  rP   rF   r  rQ   tmp_dirnlp2doc2textsbatch_deps_1batch_deps_2no_batch_depss                      r'   test_overfitting_IOr&  \  s    )C\\)$FN' "kg//T0BKPQ??62. 	"CS!	""  I3Z A

>y
@A )v%%%$I
i.Cq6;;'!!!q6;;&   q6;;'!!!q6;;==Aq6;;==Aq6;;==A	 	#7G((1IAw||w&&&Aw||v%%%Aw||w&&&Aw||~~"""Aw||~~"""Aw||~~"""	#E 4788E?CCCLL#'CLC3688E?CCCLL#'CLCKP4Q4SY4QRSS\\3%(RMR|,}--	# 	#$ DC4QRs%   2CK8L	L	,L L8Lparser_configzspacy.TransitionBasedParser.v1F@   T)z@architecturestok2vec
state_typeextra_state_tokensr2   maxout_pieces	use_upperzspacy.TransitionBasedParser.v2c                    d|i}t               }|j                  | |      }g }t        D ]a  \  }}|j                  t	        j
                  |j                  |      |             |j                  dg       D ]  }|j                  |        c |j                         }	t        d      D ]  }
i }|j                  ||	|        y )Nr-   rs   r   r   rD   )r	   r  r  r  r   rN   r  r  rK   rL   rM   rO   )r  r'  pipe_configr  r8   r  r   r  rf   r  rP   rF   s               r'   test_parser_configsr1    s     M*K
)C\\)K\8FN' "kg//T0BKPQ??62. 	"CS!	""  I1X A

>y
@Ar)   c                  0   d} d}t               }| |d}|j                  d|      }g }t        D ]a  \  }}|j                  t	        j
                  |j                  |      |             |j                  dg       D ]  }|j                  |        c |j                         }	t        d      D ]  }
i }|j                  ||	|        d	}|j                  |      }|g}|j                  |      }|j                  |      \  }}t        t        |            D ]|  }|j                  D ]*  }|d
   ||f   }d
t         z
  |cxk  rdt         z   k  r'J  J  t        t        |            D ]*  }
|d
   ||
f   }d
t         z
  |cxk  rdt         z   k  r'J  J  ~ y )Nr   r  
beam_widthbeam_densityr  r/  r   r:   rD   r  r   r   )r	   r  CONFLICTING_DATAr  r   rN   r  r  rK   rL   rM   rO   predictscored_parsesrh   rq   eps)r4  r5  r  rs   r8   r  r   r  rf   r  rP   rF   r  rQ   docsbeamshead_scoreslabel_scoresjlabellabel_score
head_scores                         r'   test_beam_parser_scoresrB    s   JL
)C $F \\-\7FN- "kg//T0BKPQ??62. 	"CS!	""  I 2Y A

>y
@A
 %I
,,y
!C5DNN4 E & 4 4U ;K3s8_ 4]] 	5E&q/1e*5Ks7k4QW44444	5 s3x 	4A$QA/Js7j3AG33333	4	4r)   c                  6   t               } d}d}||d}| j                  d|      }g }t        D ]a  \  }}|j                  t	        j
                  | j                  |      |             |j                  dg       D ]  }|j                  |        c | j                         }	t        d      D ]  }
i }| j                  ||	|        d   dk  sJ d	}| j                  |      g}|j                  |      }|j                  |      \  }}|d
   }|d
   }|d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ t#               5 }| j%                  |       t'        j(                  |      }|j                  |      g}|j+                  d      }|j                  |      }|j                  |      \  }}|d
   }|d
   }|d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ |d   t        j                  dt               k(  sJ 	 d d d        y # 1 sw Y   y xY w) Nr   r  r3  r  r/  r   r
  rD   r  r   )r   r   g      ?)abs)r   r   g        )r   r   )r   r   )r   r   )r   r   )rC   r   )rC   r   )rC   r   )r   r   )r   r   )r   r   )r   r   )r   r   )r   r   )rC   r   )rC   r   )rC   r   )r	   r  r  r  r   rN   r  r  rK   rL   rM   rO   r7  r8  pytestapproxr9  r   r  r   r  get_pipe)r  r4  r5  rs   r8   r  r   r  rf   r  rP   rF   r  r:  r;  r<  r=  r  r   docs2parser2beams2head_scores2label_scores2s                           r'   test_beam_overfitting_IOrM    s   
)CJL $F \\-\7FN' "kg//T0BKPQ??62. 	"CS!	""  I3Z A

>y
@A - 6)))$ILL#$DNN4 E & 4 4U ;Ka.K?L%s)DDDD$cs(CCCC%s)DDDD%s)DDDD$cs(CCCC%s)DDDD%s)DDDD$cs(CCCC%s)DDDDv&--"====v&--"====v&--"====v&--"====v&--"====v&--"====v&--"====v&--"====v&--"==== 
 C7G((1y)*--.'&-&;&;F&C#m#A%a(\*fmmCS.IIII[)V]]3C-HHHH\*fmmCS.IIII\*fmmCS.IIII[)V]]3C-HHHH\*fmmCS.IIII\*fmmCS.IIII[)V]]3C-HHHH\*fmmCS.IIIIF#v}}Sc'BBBBF#v}}Sc'BBBBF#v}}Sc'BBBBF#v}}Sc'BBBBF#v}}Sc'BBBBF#v}}Sc'BBBBF#v}}Sc'BBBBF#v}}Sc'BBBBF#v}}Sc'BBBB9C C Cs   K/ZZ):rE  numpy.testingr   	thinc.apir   spacyr   r   spacy.attrsr   r   spacy.lang.enr	   spacy.pipeliner
   spacy.pipeline.dep_parserr   spacy.pipeline.tok2vecr   spacy.tokensr   spacy.trainingr   spacy.vocabr   r   r   r  r6  r  r9  fixturer+   r8   r6   markissuerk   rt   rw   xfailrz   r   skipparametrizer   r   r   r   r   r   r  r  r&  slowr1  rB  rM  r%   r)   r'   <module>r`     sc    &    ! ! + : 8  "  : 	1*S	
 	$'D	

( 	$'D	
 	$'E	
 & 	$3	
 
 7 7  ,( 4
) 
) 4
) 
) 4
% 
% 4/*) + )" J   G9+. / J  *# J  F!F!R&"	& x&?@ A2 x&?@/. A/.f x&?@ =I^nv  OT  fh  {|  KO  
P<I^nv  OT  fh  {|  KO  
P	A A A#4LOCr)   