iA ddlZddlZddlZddlZddlZddlZddlmZddlm Z gejdejjejdejjejdejjejdejjd ejd ejjejd ejjejd ejjejd ejjejdejjddejdejjejdejjejdejjejdejjdejdejjejdejjejdejjejdejjdejdejjejdejjejdejjejdejjejdejjejdejjejd ejjejd!ejjd"ejd#ejjejd$ejjejd%ejjejd&ejjejd'ejjejd(ejjejd)ejjejd*ejjejd+ejjejd,ejjejd-ejjejd.ejjejd/ejjejd0ejjejd1ejjZ ejjd2e d3Zd4Zd5Zej$j&d=d6ej$j(d7ed8efd9Zejj0ejjd2e ej2e:d2ed;ed8dfd<Zy)>N) Tokenizer)get_lang_classfr)marksafarbgbnbocacsdadeelenesetfafigahehihrhuiditknlbltlvnbnlplptrosiskslsqsrsvtatetltrtturkmrlangcHt|j}tjd|d}|jD]V}||Dcgc]}|j r|j }}|j|Dcgc]}|d }}||k(rVJycc}wcc}w)Nz spacy.lang.z .examples)r tokenizerpytest importorskip sentencesis_spacetextexplain)r3r6examplessentencettokens debug_tokenss s/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/spacy/tests/tokenizer/test_explain.pytest_tokenizer_explainrCAs$t$&00I""[i#@AH&&&"+H"5HQQZZ!&&HH&/&7&7&AB!B B%%%&HBs BB BcNtjd}tjd}dddigi}t|||j|j}|dDcgc]}|j }}|j dDcgc]}|d }}||k(sJycc}wcc}w)Nz[\.]$z[/]za.ORTH)rules suffix_searchinfix_finditerza/a.r5)recompilersearchfinditerr;r<)en_vocab suffix_reinfix_rerFr6r?r@explain_tokenss rB&test_tokenizer_explain_special_matcherrQKs 8$Izz&!H VTN# $E&&(( I (/ 0aff 0F 0$-$5$5f$=>qad>N> ^ ## #1>s B B"cdddigi}t||}d}||Dcgc]}|j}}|j|Dcgc]}|d }}||k(sJycc}wcc}w)Nz:]rE)rFz: ]r5)rr;r<)rMrFr6r;r?r@rPs rB1test_tokenizer_explain_special_matcher_whitespacerSZs} VTN# $EI D'o .aff .F .$-$5$5d$; token_pairtokens rBsentence_strategyrlfs#&((<!3!3 4A299Q< 4r>cHtj|j}tjdd|j }||Dcgc]}|j }}|j|Dcgc]}|d }}||k(sJ|d|d|ycc}wcc}w)z Tests whether output of tokenizer.explain() matches tokenizer output. Input generated by hypothesis. lang (str): Language to test. text (str): Fuzzily generated sentence to tokenize. z\s+r]r5z, N)spacyblankr6rIsubstripr;r<)r3r>r6r?r@rAs rBtest_tokenizer_explain_fuzzyrrs!;;t,66Ivvfc8,224H'1 2aff 2F 2"+"3"3H"=>QAaD>L> \ !JfXR ~Rz#JJ !3>s B7 B))rIr_rchypothesis.strategiesr7rnspacy.tokenizerr spacy.utilrparammarkslow LANGUAGES parametrizerCrQrSrd compositeDrawFnintstrrlxfailgivenrrrBrs  %% / FLLV[[--/0/ FLLV[[--/0/ FLLV[[--/0/ FLLV[[--/0 /   /  FLLV[[--/0 / FLLV[[--/0/ FLLV[[--/0/ FLLV[[--/0/ FLLV[[--/0/  /  / FLLV[[--/0/ FLLV[[--/0/ FLLV[[--/0/  FLLV[[--/0!/ " #/ $FLLV[[--/0%/ &FLLV[[--/0'/ (FLLV[[--/0)/ *FLLV[[--/0+/ , -/ .FLLV[[--/0// 0FLLV[[--/01/ 2FLLV[[--/03/ 4FLLV[[--/05/ 6FLLV[[--/07/ 8FLLV[[--/09/ :FLLV[[--/0;/ <FLLV[[--/0=/ > ?/ @FLLV[[--/0A/ BFLLV[[--/0C/ DFLLV[[--/0E/ FFLLV[[--/0G/ HFLLV[[--/0I/ JFLLV[[--/0K/ LFLLV[[--/0M/ NFLLV[[--/0O/ PFLLV[[--/0Q/ RFLLV[[--/0S/ TFLLV[[--/0U/ VFLLV[[--/0W/ XFLLV[[--/0Y/ ZFLLV[[--/0[/ \FLLfkk..01]/  d+&,& $ $   PJ1188PsPSVP!P4+,./ Ks Kc Kd K0, Kr