is# ddlZddlmZmZmZmZmZmZmZddl m Z m Z ddl m Z mZdeedeefdZdeedeefd Zdeedeefd Zdeedeefd Zdd e d efdZd e deefdZ dd e deeeeeeeffd edeefdZd e deedeefdZd e deedeeeeeeefffdZdeedeeeeeffdZdedeeeffdZdedefdZeZeZeZy)N)DictIterableIteratorListTupleUnioncast)ErrorsWarnings)DocSpantagsreturncg}t|}|r7|jt||jt||r7|S)N)listextend _consume_os _consume_ent)routs i/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/spacy/training/iob_utils.py iob_to_biluorsAC :D  ;t$% <%&  Jcg}|D]K}||j||jdddjddd}|j|M|S)NU-B-L-I-)appendreplace)rrtags r biluo_to_iobr#sY C ; JJsO++dD!,44T4CC JJsO  Jrc#bK|r)|ddk(r |jd|r |ddk(ryyyyw)NrO)pop)rs rrrs6 47c>hhqk 47c>$>$s(//c|sgS|jd}d|ddz}d|ddz}d}|r+|d||hvr"|dz }|jd|r |d||hvr"|dd}|dk(r=t|dk(r)ttjj |d|zgSd|z}d |z}t d|dz Dcgc]}d | } }|g| z|gzScc}w) NrIrLr r"rrrr)r&len ValueErrorr E177formatrange) rr" target_in target_lastlengthlabelstartend_middles rrr!s   ((1+Cc!"g IAB-K F 47y+66!    47y+66 GE { u:?V[[//C/89 9u ~u Ul(-a!(<=1Bug,==w3%''>s7 Cdocmissingc t||jDcgc]%}|j|j|jf'c}|Scc}w)Nr9)offsets_to_biluo_tagsents start_charend_charlabel_)r8r9ents rdoc_to_biluo_tagsrB7s=  ?BxxH#..#,, 3H Hs*A cpt|d}t|D]\}}|jdk(sd||<|S)N-r;r r%)rB enumerateent_iob)r8r=itokens r_doc_to_biluo_tags_with_partialrI?sA S# .DcN5 ==A DG Krentitiesc  i}|Dcic]}|j|j}}|Dcic]%}|jt|z|j'}}|Dcgc]}d}}|D]\} } } | s|D]} | | k\s | | ksd||| <%t| | D]^} | |j vrBt t jj|| d|| d|| df| | | f| | | f|| <`|j| }|j| }||||k(r d| ||<d | ||<t|dz|D] }d | ||< d | ||<t}|D](\} } } t| | D]}|j|*|D]H}t|j|jt|zD]}||vs9|||j<Jd|vr|dk7rt|}tjtj jt|j"d kDr|j"dd d zn |j"t|d kDr|dd d zn||Scc}wcc}wcc}w)uEncode labelled spans into per-token tags, using the Begin/In/Last/Unit/Out scheme (BILUO). doc (Doc): The document that the entity offsets refer to. The output tags will refer to the token boundaries within the document. entities (iterable): A sequence of `(start, end, label)` triples. `start` and `end` should be character-offset integers denoting the slice into the original string. missing (str): The label used for missing values, e.g. if tokenization doesn’t align with the entity offsets. Defaults to "O". RETURNS (list): A list of unicode strings, describing the tags. Each tag string will be of the form either "", "O" or "{action}-{label}", where action is one of "B", "I", "L", "U". The missing label is used where the entity offsets don't align with the tokenization in the `Doc` object. The training algorithm will view these as missing values. "O" denotes a non-entity token. "B" denotes the beginning of a multi-token entity, "I" the inside of an entity of three or more tokens, and "L" the end of an entity of two or more tokens. "U" denotes a single-token entity. EXAMPLE: >>> text = 'I like London.' >>> entities = [(len('I like '), len('I like London'), 'LOC')] >>> doc = nlp.tokenizer(text) >>> tags = offsets_to_biluo_tags(doc, entities) >>> assert tags == ["O", "O", 'U-LOC', "O"] rDr%rrr )span1span2Nrrrr2z...)textrJ)idxrGr+r/keysr,r E103r.getsetaddstrwarningswarnr W030rO)r8rJr9tokens_in_entsrHstartsendsr6biluor>r?r3s token_index start_token end_tokenrG entity_charsent_strs rr<r<Gs<CEN.1 2Ueii 2F 29< =EIIE "EGG + =D = QS E '/4# He + ?q8|'*E&)$ + %Z: L ."5"5"77$ ** .{ ;A > .{ ;A > .{ ;A ># $.x"? +  0:8U.K{+ L!**Z0K*I&9+@)++-eWE+&+-eWE+&";?I>0%'w<a0)+E7|E)$;4>5L'/ # Hez8, A   Q   %uyy%))c%j"89 %AL  %%E%''N %  e|3h- MM .1#((mb.@SXXcr]U*chh14W1B"- !  Lk3 = sI6*I; Jc|t|}g}|D])\}}}t|||dz|}|j|+|S)aEncode per-token tags following the BILUO scheme into Span object, e.g. to overwrite the doc.ents. doc (Doc): The document that the BILUO tags refer to. tags (iterable): A sequence of BILUO tags with each tag describing one token. Each tag string will be of the form of either "", "O" or "{action}-{label}", where action is one of "B", "I", "L", "U". RETURNS (list): A sequence of Span objects. Each token with a missing IOB tag is returned as a Span with an empty label. rr3)tags_to_entitiesrr )r8r token_offsetsspansr3 start_idxend_idxspans rbiluo_tags_to_spansrlsP%T*M E%2!y'CGaKu= T Lrct||}|Dcgc]%}|j|j|jf'c}Scc}w)aEncode per-token tags following the BILUO scheme into entity offsets. doc (Doc): The document that the BILUO tags refer to. tags (iterable): A sequence of BILUO tags with each tag describing one token. Each tags string will be of the form of either "", "O" or "{action}-{label}", where action is one of "B", "I", "L", "U". RETURNS (list): A sequence of `(start, end, label)` triples. `start` and `end` will be character-offset integers denoting the slice into the original string. )rlr>r?r@)r8rrhrks rbiluo_tags_to_offsetsrns7 T *EFK LdT__dmmT[[ 9 LL Ls*>c g}d}t|D]]\}}||jdr|d}|jd||f4|jdrF|jdr<|Zttj j dt|d|dz|jdr|j|d d||f|jd r|}|jd rV|9ttj j d t|d|dz|j|d d||fd}7ttjj | |S) zxNote that the end index returned by this function is inclusive. To use it for Span creation, increment the end by 1.NrDr%r(r)r4rUr Br)r*) rE startswithr r,r E067r.rE068)rrJr4rGr"s rrfrfsdH ED/:3 ;#..- Q + ^^C  ^^C } KK&&StDz'AE7J&K^^C OOSWaO , ^^C E ^^C } KK&&StDz'AE7J&K OOSWeQ/ 0EV[[//C/89 95:6 Orr3c\ttttf|jddSNrDr)r rrVsplitres rsplit_bilu_labelrys" c3hS!!4 55rc,|jdddSrw)rxres rremove_bilu_prefixr{s ;;sA q !!r)r%) rWtypingrrrrrrr errorsr r tokensr rrVrr#rrrBrIintr<rlrnrfryr{offsets_from_biluo_tagsspans_from_biluo_tagsbiluo_tags_from_offsetsrrrsEEE%x}cx}cd3iHSM (tCy(T#Y(,3cTWT T sCsCx'@!ABTMPT #YTnS $t*&M MSMM %S%S/) *+M" 8C= T%S# 2F-G F6C6E#s(O6"c"c" 0+/r