
    	iq                        d dl Z d dlZd dlmZ d dlmZ d dlmZmZ d dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ d	d
gZ ej&                  e      xZZ ed       G d d	             Zde	j0                  j2                  dedefdZ ed      	 	 	 	 ddddede	j0                  j2                  deegef   deeeef      dee   dee   dedee   fd       Zy)    N)OrderedDict)Callable)AnyOptional)compatibility)lazy_format_graph_code)GraphModule)Node	Partitionsplit_moduleT)is_backward_compatiblec                   $    e Zd ZdefdZdefdZy)r   namec                     || _         d| | _        g | _        i | _        i | _        i | _        i | _        t        j                  j                  j                         | _	        i | _        i | _        y )Nsubmod_)r   submod_name
node_namesinputsoutputsdependencies
dependentstorchfxgraphGraphenvironmenttargets)selfr   s     m/var/www/vps2.regionflexible.com/Desarrollo/venv/lib/python3.12/site-packages/torch/fx/passes/split_module.py__init__zPartition.__init__   sc    	$TF+%'')(*-/+-+088>>+?+?+A
-/')    returnc                     d| j                    d| j                   d| j                   d| j                   d| j                   d| j
                   S )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depended on: z,
 partition dependents: )r   r   r   r   r   r   )r   s    r   __repr__zPartition.__repr__!   sb    TYYK  ' (} % '((,(9(9': ;&&*oo%68	
r!   N)__name__
__module____qualname__strr    r$    r!   r   r   r      s    
*S 
*
# 
r!   modqualnamer"   c                     | }|j                  d      D ])  }t        ||      st        d| d      t        ||      }+ |S )N.zNode target z not found!)splithasattrAttributeErrorgetattr)r*   r+   attr_valatoms       r   _get_attr_from_qualnamer4   ,   sP    Hs# +x& <z!EFF8T*+ Or!   )partition_affixmroot_msplit_callbackqualname_mapkeep_original_orderkeep_original_node_namekeep_original_input_namer5   c                8    2?@ABCDEFGHI t         j                  dt        d d             dt        dt        t
        t        f   dt        t
        t        j                  j                  j                  f   f@ fd}d	d
l
}	i Gi Ei Idt        dt        t           fGIfdHGfd}
t        j                  j                  t        j                  j                  t        j                  j                   g}t#               }t#               }i }d
}t%               } j&                  j(                  D ]  22j*                  j-                  d      x}st/        |t        j0                  t        j2                  f      rIt/        |j4                  j6                  x}|	j8                        r|Ivr2I|j4                  j6                  <   2j:                  dv r |
2       2j:                  dk(  r2j<                  |v r2j<                  t        j                  j                   u rt?        2j@                        dk7  r!tC        dt?        2j@                               t/        2j@                  d	   tD              s$tC        dtG        2j@                  d	                2}t%         2      h      ||<   n/2j<                  t        j                  j                  u rStI        d 2j@                  D              stC        d      |jK                  2       t%         2      h      |2<   d
|2<   n2j<                  t        j                  j                  u rt?        2j@                        dk7  r!tC        dt?        2j@                               |2j@                  d	      jK                   2             |jM                  2j@                  d	          2|2j@                  d	   <   |||   jK                   2             |D ]  }||   jK                   2               tI        d |jO                         D              stC        d      |jQ                         D ci c]  \  }}|tS        |       }}}|jQ                         D ci c]  \  }}|tS        |       }}}tT        jW                  tX        jZ                        r,tT        j                  d|       tT        j                  d|       tE        |      xs tE        |      }d} j&                  j(                  D ]  22E2j\                  <   2j:                  dv r"2j:                  dk(  r;t        j                  j&                  j_                  2j@                  d	   Hfd       l|r  2      }||kD  rtC        d | d!|       |}2j<                  |vst        j                  j&                  j_                  2j@                  2Hfd"       t        j                  j&                  j_                  2j`                  2Hfd#        tc        Gje                               }g }GjQ                         D ],  \  }Ft?        Fjf                        r|ji                  |       . g }|rw|jk                         }|ji                  |       G|   jl                  D ]A  }G|   jf                  jk                  |       G|   jf                  r1|ji                  |       C |rwt?        |      t?        G      k7  rto        d$      ||fD ]  }|jQ                         D ]  \  2} t?        |       d	k(  rtC        d%      2Gt        | d	            jp                  2<   | dd
 D ]  }!Gt        |!         FFj&                  js                  2j:                  2j<                  tu        d& 2j@                  D              i 2jF                  '      }"2j*                  jw                         |"_        |"Fjp                  2<      |D ];  }G|   Fi Dd	AFjx                  D ]  CEC   }#ACDEFfd(}$|#j:                  d)k(  rt/        |#j<                  t
              s!tC        d*tG        |#j<                               t{         |#j<                        }%t/        |%t        j|                  j~                        r?Fj&                  j                  |#j<                        }&|%Fj                  |#j<                  <   n |$       }&n |$       }&EC   j*                  jw                         |&_        |&Fjp                  EC   <    DF_<        >  j&                  j(                  D ]  2t        2d+      sG2j                     FFjp                  Bt        j                  j&                  j_                  2j@                  Bfd,      }'t        j                  j&                  j_                  2j`                  Bfd-      }(2j:                  d.vr2j<                  })nct{         2j<                        }*2j<                  j                  d/d0      })|*Fj                  |)<   | Fj                   d/|) }+2j<                  ||+<   t/        |'tt              stC        d1tG        |'             t/        |(t              stC        d2tG        |(             r2j\                  nd
},Fj&                  js                  2j:                  |)|'|(2jF                  |,3      }"2j*                  jw                         |"_        |"Fjp                  2<    |fD ]  }t        |      D ]  2|2   } t?        |       d	k(  rtC        d4      | d
d D ]  }!Gt        |!         F|2   }-|-tC        d5      Fj&                  js                  |-j:                  |-j<                  Fjp                  2   fi |-jF                  '      }"|-j*                  jw                         |"_           i }.i ?t        j                  j&                  j                         @i }/|s) j&                  j(                  D ]  2 |2?|/      \  ?}/ n* j&                  j(                  D ]  22|.2j\                  <    |s|n|}0t%               }1 j&                  j(                  D 2cg c]  }2|2j:                  d6k(  s|2 }3}2|0D ]'  }G|   Ftu        EFfd7Fj                  D              }4t?        |4      }5|5dk(  rFj&                  j                  |4d	          n<|5dkD  rFj&                  j                  |4       nFj&                  j                  d8       |rtFjx                  D 6cg c]  }6|6|3vr|.|6    }7}6|3D ]%  22|1v r |2?|/      \  ?}8|1jK                  2       ' |7D ]%  22|1v r |2?|/      \  ?}/|1jK                  2       ' t        j                  j                  j                  Fj                  Fj&                        |/Fj                  <   @j                  Fj                  tu        ?fd9Fjx                  D                    }9t?        Fj                        }:|:dkD  rZt        j                  j                  j                  |9      };t        Fj                        D ]  \  }<}=|;|<   j4                  ?|=<     |:dk(  s|9?t        t        Fj                              <   * |r*?s( j&                  j(                  D ]  2 |2?|/      \  ?}/  j&                  j(                  D ][  22j:                  dk(  s@j                  t        j                  j&                  j_                  2j@                  d	   ?fd:             ] t        j                  j                  j                  |/@      }>t         j                  dt        d;|>d             |>S c c}}w c c}}w c c}2w c c}6w )<a  
    Creates subgraphs out of main graph

    Args:
        m (GraphModule): Graph module to split
        root_m (torch.nn.Module): root nn module. Not currently used. Included
            because the root nn module is usually transformed via
            torch.fx._symbolic_trace.symbolic_trace (see example below)
        split_callback (Callable[[Node], int]): Callable function
            that maps a given Node instance to a numeric partition identifier.
            split_module will use this function as the policy for which operations
            appear in which partitions in the output Module.
        qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
            mapping from new target names in the module after split to old target
            names in the original module.
        keep_original_order: Optional[bool]: keep the original order of the GraphModule
            or use the Topological order of the new constructed GraphModule
        keep_original_node_name: Optional[bool]: If the partitioned graphs should
            have the same node names as the original graph.
        keep_original_input_name: bool: If the partitioned graphs should
            have the same input names as the original graph.
        partition_affix: Optional[str]: If specified, the submodules' names will contain
            the affix, e.g. "submod_<affix>_<idx>".

    Returns:
        GraphModule: the module after split.

    Example:

        This is a sample setup:

            import torch
            from torch.fx._symbolic_trace import symbolic_trace
            from torch.fx.graph_module import GraphModule
            from torch.fx.node import Node
            from torch.fx.passes.split_module import split_module

            class MyModule(torch.nn.Module):
                def __init__(self) -> None:
                    super().__init__()
                    self.param = torch.nn.Parameter(torch.rand(3, 4))
                    self.linear = torch.nn.Linear(4, 5)

                def forward(self, x, y):
                    z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                    w = self.linear(y).clamp(min=0.0, max=1.0)
                    return z + w

            # symbolically trace model
            my_module = MyModule()
            my_module_traced = symbolic_trace(my_module)

            # random mod partitioning
            partition_counter = 0
            NPARTITIONS = 3

            def mod_partition(node: Node):
                global partition_counter
                partition = partition_counter % NPARTITIONS
                partition_counter = (partition_counter + 1) % NPARTITIONS
                return partition

            # split module in module with submodules
            module_with_submodules = split_module(
                my_module_traced, my_module, mod_partition
            )

        Output looks like this. Original graph is broken into partitions

            > print(module_with_submodules)
            GraphModule(
                (submod_0): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_1): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_2): GraphModule()
            )

            def forward(self, x, y):
                param = self.param
                submod_0 = self.submod_0(x, param, y);  x = param = y = None
                getitem = submod_0[0]
                getitem_1 = submod_0[1];  submod_0 = None
                submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
                getitem_2 = submod_1[0]
                getitem_3 = submod_1[1];  submod_1 = None
                submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
                return submod_2

        Output of split module is the same as output of input traced module.
        This is an example within a test setting:

            > orig_out = my_module_traced(x, y)
            > submodules_out = module_with_submodules(x, y)
            > self.assertEqual(orig_out, submodules_out)
            True
    z%szpre split_moduleT)colorednodebase_mod_envbase_mod_attrsc                    | j                   dk(  r t        | j                        dkD  r| j                  d   nt        j                  j
                  }rX|t        j                  j
                  u rdn|f}j                  d| j                  || j                        || j                  <   n5j                  | j                  | j                  |      || j                  <   | j                  j                         || j                     _        ||fS | j                   dk(  rj                  | j                        || j                  <   | j                  j                         || j                     _        t        | j                  t              s!t!        dt        | j                               t#        | j                        }||| j                  <   ||fS )Nplaceholderr   r)   )args	type_expr)rE   default_valueget_attrExpected str target, got )oplenrD   inspect	Signatureemptycreate_noder   typerC   targetmetacopyrG   
isinstancer(   AssertionErrorr4   )	r?   r@   rA   rF   rD   r2   base_mod_graphr;   r6   s	         r   construct_graphz%split_module.<locals>.construct_graph   s   
 77m# #DII 2		!8I8I8O8O  ''7+<+<+B+BBBHX  +9*D*D!II"ii	 +E +TYY' +9*D*DKK"ii"/ +E +TYY'
 ,099>>+;L#( ^++ WW
"&4&=&=dkk&JL#+/99>>+;L#(dkk3/$'@dkkAR@S%TUU.q$++>H*2N4;;'^++r!   r   Ndef_nodeuse_nodec                    ddl m} t        | dd       }t        |dd       }t        j	                  d| j
                  |||j
                  nd|       ||k7  r||G|   }|j                  j                  | j
                         ||j                  j                  |       |/|   }|j                  j                  | j
                         | j                  j                  d      x}t         ||      t              D ]  }|   }	|j                  j                  |	j
                         |   j                  dk7  s@t        |	dd       }
|
P|
   }|j                  j                  |	j
                         |j                  j                  |       |j                  j                  |
        ||j                  j                  |       y y y y )	Nr   )free_symbols_fx_partitionz*record_cross_partition_use %s (%s) %s (%s)-example_value)keyrC   )%torch.fx.experimental.symbolic_shapesrZ   r1   logdebugr   r   
setdefaultr   r   rQ   getsortedr(   rI   r   )rW   rX   rZ   defineduseddef_partitionuse_partitiondef_valss_node	s_defineds_def_partition
partitionssymbol_to_nodes               r   record_cross_partition_usez0split_module.<locals>.record_cross_partition_use   s   F(OT:x$7		8MM%1HMMs	
 d?" *7 3%%00?#!,,77= *4 0$$//>  (}}00AAGN#L$9sC Q!/!2%,,77D)!,//=@ )0(NI(42<Y2G / 7 7 B B6;; O / : : E Ed K - : : E Ei P#Q$ &!..99'B '3   r!   c                 6    |       }t        |      }dj                  |g      }t        j                  d| j                  |       j                  |      }|t        |      x|<   }|j                  j                  | j                         || _	        y )N_z*instantiate_node_partition_mapping %s (%s))
r(   joinr`   ra   r   rc   r   r   appendr[   )r?   partition_idxpartition_name	partitionr5   rn   r8   s       r   "instantiate_node_partition_mappingz8split_module.<locals>.instantiate_node_partition_mapping  s    &t,]+& !XX&GHN		8$))^	

 NN>2	5>~5NNJ~&##DII.+r!   r]   )rC   rG   outputcall_function   z*Expected 1 arg for _set_grad_enabled, got zExpected bool arg, got c              3   >   K   | ]  }t        |t                 y wN)rS   r
   .0args     r   	<genexpr>zsplit_module.<locals>.<genexpr>U  s     Jz#t44Js   z3Expected all args to be python constants, not Nodesz'Expected 1 arg for _exit_autocast, got c              3   $   K   | ]  }|d u 
 y wr}   r)   )r   vs     r   r   zsplit_module.<locals>.<genexpr>k  s     >q}>s   zautocast must exitzautocast_regions: %szgrad_regions: %s)rC   rG   ry   c                      | d       S r}   r)   )nrp   s    r   <lambda>zsplit_module.<locals>.<lambda>  s    (B1d(K r!   zSautocast or set_grad_enabled require monotonically increasing partitions: highest: z, this node's: c                      |       S r}   r)   rW   r?   rp   s    r   r   zsplit_module.<locals>.<lambda>  s    ,FxQU,V r!   c                      |       S r}   r)   r   s    r   r   zsplit_module.<locals>.<lambda>  s    .HSW.X r!   z cycle exists between partitions!z%Expected at least one region for nodec              3       K   | ]  }|  y wr}   r)   r~   s     r   r   zsplit_module.<locals>.<genexpr>  s     8ss8s   )rI   rP   rD   kwargsrE   c                      r} n
d } dz  j                   j                  |    j                        }d <   |S )Narg_r{   )rE   )r   rC   rO   )r   rC   counterinpr<   
new_inputs
orig_nodesrw   s     r   add_placeholderz%split_module.<locals>.add_placeholder  s\    +D "'+DqLG'oo99(o22 :  #'
3""r!   rG   rH   r[   c                     |    S r}   r)   r   r   s    r   r   zsplit_module.<locals>.<lambda>  s    TU r!   c                     |    S r}   r)   r   s    r   r   zsplit_module.<locals>.<lambda>  s    {1~ r!   )call_modulerG   r-   rr   z&Expected tuple for gathered_args, got z'Expected dict for gathered_kwargs, got )rI   rP   rD   r   rE   r   zExpected at least one regionzMissing exit noderC   c              3   B   K   | ]  }j                   |        y wr}   )r   )r   r   r   rw   s     r   r   zsplit_module.<locals>.<genexpr>Z  s&      
8<I!!*T"23
s   r)   c              3   (   K   | ]	  }|     y wr}   r)   )r   r   r@   s     r   r   zsplit_module.<locals>.<genexpr>  s     B,t$Bs   c                 "    | j                      S r}   )r   )r   r@   s    r   r   zsplit_module.<locals>.<lambda>  s    |AFF?S r!   zpost split_module)Pr`   ra   r   r
   dictr(   r   r   graph_moduler	   sympyr   amp_enter_autocast_exit_autocast_C_set_grad_enabledr   setr   nodesrQ   rc   rS   SymIntSymFloatr?   exprSymbolrI   rP   rJ   rD   rT   boolrO   alladdremovevaluesitemsrd   _LOGGERisEnabledForloggingDEBUGr   map_argr   listkeysr   rt   popr   RuntimeErrorr   rN   tuplerR   r   r4   nnModulerG   r   r/   r[   replacer   reversedr   r   ry   r   proxyProxy	enumeratenextiter)Jr6   r7   r8   r9   r:   r;   r<   r5   rV   r   rx   GLOBAL_STATE_NODESgrad_regionsautocast_regionsautocast_exitsactive_gradactive_autocastsvals0akr   assert_monotonically_increasinghighest_partitionpidoriginal_partition_orderroot_partitionsrv   sorted_partitionsroot_partition	dependentregions_mappingregionsrnew_node	orig_noder   	orig_attrrC   gathered_argsgathered_kwargsrP   target_attrr+   r   	exit_nodeorig_mod_envrA   construct_order_partitionsalready_constructed_attr_nodesr?   original_orderoutput_valsnum_output_valsr^   orig_mod_attr_nodes_based_mod_attrs
output_valnum_outputsoutput_val_proxyioutput_nameretr@   rU   r   r   r   r   r   rw   rn   rp   ro   sJ   ` `  ```                                          `            @@@@@@@@@@@r   r   r   6   s   ` II11dC
!,!,39o!, S%(("7"7"C"CCD!,F ')J"$J/1N/CT /CXd^ /Cb,. 			!!		  "" 1<L 5@M13NKu 5: IIMM/22S?3u~~ >?.2=.(,0N388==)77;;*4077o%$++9K*K{{ehh888tyy>Q&(DS^DTU  "$))A,5(+B4		RSCUBV)WXX",/1E0F,G[)		 9 99J		JJ(M  !$$T*),nT.B-C)D &'+t$		 8 88tyy>Q&(A#dii.AQR  !1.22>$3GH ''		!5/3tyy|,"%)).*>?! 	:AQ##N4$89	:i5:n >n&;&;&=>>122 2B1G1G1IJA6!9JJ-9-?-?-ABTQAvayLBLBGMM*,.>?(,7&*+;&<&R\@R#   $
499 771177hHHNN""		!K * &C 3&$,,=+>ocUT  !$ ;;00HHNN""		V HHNN""X9@  $JOO$56!#O%/%5%5%7 3!	9))*"">23
 $&
(,,.  0#N3>> 	2Iy!..22>Bi(55&&y1	2  Z0=>> -l; 7,224 	7MD'7|q $%LMM<@Js71:'33D9 QR[ 7&s1v.	$??66ww;;8dii88"ii 7  IINN$  /7	%%d+7	77. , )&~.	&(
## "	AC"3I# # ||z)!)"2"2C8(3D9I9I4J3KL  4Ay7G7GH	i9"+//":":9;K;K"LK:CI%%i&6&67"1"3K-/)#3388:K5@I!!*S/2E"	AF &	S)&X  *34)"4#5#56I $//K!HHNN22499>VWM#hhnn445O ww995aE,,S#6,7	!!&)+ #,"7"7!8&BH-1[[L*mU3$<T-=P<QR  ot4$=d?>S=TU  !8499TD 2277"&)) 3 H !IINN,HM*2I!!$'U*3Z -- _- 	D%d+G7|q $%CDDSb\ &s1v.	*40	$()<==$??66 ||$++#//57'nn 7  NN'') 	. %'L$&L+088>>+?+?+ANCENGGMM 	D+:lN,(L.	 GGMM 	+D&*L#	+ "5:R  &)U" ()ww}}Qt=8PdQNQ4 <E~.	  
@I@Q@Q
 

 k*aOO"";q>2q OO"";/ OO""2& %++/n, S!/ / ' 9991@,2.. /22489 , 999/>,0,n /22489 160E0E0Q0Qy1
y,,-
 $//!!B1A1ABB


 )++,?$xx~~33J?"+I,=,="> E;,<Q,?,D,D[)EA:DLd9#4#4567y<ED <GGMM 	D+:lN,(L.	  77h!!&&tyy|5ST ((


+
+NN
KCII2CF Jw	 KBF R,/s#   &A@A@A@%A@A@)NFFT) rK   r   collectionsr   collections.abcr   typingr   r   r   torch.fx._compatibilityr   torch.fx._utilsr   torch.fx.graph_moduler	   torch.fx.noder
   __all__	getLoggerr%   r`   r   r   r   r   r(   r4   intr   r   r   r)   r!   r   <module>r      s:     # $    1 2 -  
'!!!(+ +g d+
 
 ,
0 C C  d+
 .2*/.3%)s	 &*s	s	HHOOs	 dVS[)s	 4S>*	s	
 "$s	 &d^s	 #s	 c]s	 ,s	r!   