o
    j9:jeq                     @   s4  d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZ d
dgZee ZZe
ddG dd
 d
ZdejjdedefddZe
dd				d ddddedejjdeegef deeef dB dedB dedB dededB defddZ dS )!    N)OrderedDict)Callable)Any)compatibility)_make_graph_module)lazy_format_graph_code)GraphModule)Node	Partitionsplit_moduleT)is_backward_compatiblec                   @   s(   e Zd ZdefddZdefddZdS )r
   namec                 C   sN   || _ d| | _g | _i | _i | _i | _i | _tjj	
 | _	i | _i | _d S )Nsubmod_)r   submod_name
node_namesinputsoutputsdependencies
dependentstorchfxgraphGraphenvironmenttargets)selfr    r   c/home/nk/hobo-godmode/plappi-mvp/.venv/lib/python3.10/site-packages/torch/fx/passes/split_module.py__init__   s   
zPartition.__init__returnc                 C   s4   d| j  d| j d| j d| j d| j d| j S )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depended on: z,
 partition dependents: )r   r   r   r   r   r   )r   r   r   r   __repr__"   s   
zPartition.__repr__N)__name__
__module____qualname__strr   r    r   r   r   r   r
      s    modqualnamer   c                 C   s<   | }| dD ]}t||std| dt||}q|S )N.zNode target z not found!)splithasattrAttributeErrorgetattr)r%   r&   attr_valatomr   r   r   _get_attr_from_qualname-   s   
r.   F)partition_affixtuple_returnmroot_msplit_callbackqualname_mapkeep_original_orderkeep_original_node_namekeep_original_input_namer/   r0   c          9   
      s
  t dtddd dtdtttf dtttjjj	f ffdd	}	d
dl
}
i i i dtdtdB ffddfdd}tjjtjjtjjg}t }t }i }d}t }jjD ]		jd }durt|tjtjfrt|jj }|
jr|vr	|jj< 	jdv rqa|	 	jdkr6	j|v r6	jtjju rt	jdkrt dt	j t	jd
 t!st dt"	jd
  	}t	h||< n^	jtjju rt#dd 	jD st d|$	 t	h|	< d|	< n5	jtjju r6t	jdkrt dt	j |	jd
  $	 |%	jd
  	|	jd
 < |durD|| $	 |D ]}|| $	 qFqat#dd |& D sdt ddd |' D }d d |' D }t()t*j+rt(d!| t(d"| t!|pt!|}d#}jjD ]a			j,< 	jd$v rq	jd%krtjj-	jd
 fd&d' q|rֈ	}||krt d(| d)| |}	j|vrtjj-	j	fd*d' tjj-	j.	fd+d' qt/0 }g }' D ]\}tj1s|2| qg }|rD|3 }|2| | j4D ]}| j13| | j1s?|2| q*|st|tkrQt5d,||fD ]T}|' D ]L\	}t|d
krjt d-	t|d
  j6	< |dd D ]*} t|  jj7	j	jt8d.d 	jD i 	j"d/}!	j9 |!_|!j6	< q{q[qU|D ]m}| i d
j:D ][ }"fd0d1}#|"jd2krt|"jtst d3t"|"j t;|"j}$t|$tj<j=rj>|"j}%|$j?|"j< n|# }%n|# }% j9 |%_|%j6 < q_:qjjD ]	t@	d4r	jA j6tjj-	jfd5d'}&tjj-	j.fd6d'}'	jd7vrR	j}(n$t;	j})	jBd8d9}(|)j?|(< |durvjC d8|( }*	j||*< t|&t8st d:t"|& t|'tst d;t"|' r	j,nd}+jj7	j|(|&|'	j"|+d<}!	j9 |!_|!j6	< q|fD ]O}tD|D ]G	|	 }t|d
krt d=|dd# D ].} t|  |	 },|,du rt d>jj7|,j|,jj6	 fi |,j"d/}!|,j9 |!_qאqqi 
i  tjjE i }-|s*jjD ]	|		 |-\ }-qnjjD ]		
	j,< q.|s<|n|}.t }/d?d@ jjD |.D ]}| t8fdAdjFD }0t|0dkrr|srjG|0d
  njG|0 |r
fdBd@j:D }1D ]		|/v rq|		 |-\ }2|/$	 q|1D ]		|/v rq|		 |-\ }-|/$	 qtHj?j|-jC< IjCt8 fdCdj:D }3tjF}4|4dks|4dkr|rtjjJK|3}5tLjFD ]\}6}7|5|6 j |7< qqL|4dkr|3 tMtNjF< qL|r( s(jjD ]	|		 |-\ }-qjjD ]		jd%krFGtjj-	jd
  fdDd' q,tH|-}8t dtdE|8dd |8S )Fa  
    Creates subgraphs out of main graph

    Args:
        m (GraphModule): Graph module to split
        root_m (torch.nn.Module): root nn module. Not currently used. Included
            because the root nn module is usually transformed via
            torch.fx._symbolic_trace.symbolic_trace (see example below)
        split_callback (Callable[[Node], int]): Callable function
            that maps a given Node instance to a numeric partition identifier.
            split_module will use this function as the policy for which operations
            appear in which partitions in the output Module.
        qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
            mapping from new target names in the module after split to old target
            names in the original module.
        keep_original_order: Optional[bool]: keep the original order of the GraphModule
            or use the Topological order of the new constructed GraphModule
        keep_original_node_name: Optional[bool]: If the partitioned graphs should
            have the same node names as the original graph.
        keep_original_input_name: bool: If the partitioned graphs should
            have the same input names as the original graph.
        partition_affix: Optional[str]: If specified, the submodules' names will contain
            the affix, e.g. "submod_<affix>_<idx>".
        tuple_return: bool: If True, submodule outputs are always wrapped in a tuple,
            even when there is only a single output value.  This makes all subgraphs
            conform to the convention expected by ``torch._inductor.compile_fx``.

    Returns:
        GraphModule: the module after split.

    Example:

        This is a sample setup:

            import torch
            from torch.fx._symbolic_trace import symbolic_trace
            from torch.fx.graph_module import GraphModule
            from torch.fx.node import Node
            from torch.fx.passes.split_module import split_module

            class MyModule(torch.nn.Module):
                def __init__(self) -> None:
                    super().__init__()
                    self.param = torch.nn.Parameter(torch.rand(3, 4))
                    self.linear = torch.nn.Linear(4, 5)

                def forward(self, x, y):
                    z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                    w = self.linear(y).clamp(min=0.0, max=1.0)
                    return z + w

            # symbolically trace model
            my_module = MyModule()
            my_module_traced = symbolic_trace(my_module)

            # random mod partitioning
            partition_counter = 0
            NPARTITIONS = 3

            def mod_partition(node: Node):
                global partition_counter
                partition = partition_counter % NPARTITIONS
                partition_counter = (partition_counter + 1) % NPARTITIONS
                return partition

            # split module in module with submodules
            module_with_submodules = split_module(
                my_module_traced, my_module, mod_partition
            )

        Output looks like this. Original graph is broken into partitions

            > print(module_with_submodules)
            GraphModule(
                (submod_0): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_1): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_2): GraphModule()
            )

            def forward(self, x, y):
                param = self.param
                submod_0 = self.submod_0(x, param, y);  x = param = y = None
                getitem = submod_0[0]
                getitem_1 = submod_0[1];  submod_0 = None
                submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
                getitem_2 = submod_1[0]
                getitem_3 = submod_1[1];  submod_1 = None
                submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
                return submod_2

        Output of split module is the same as output of input traced module.
        This is an example within a test setting:

            > orig_out = my_module_traced(x, y)
            > submodules_out = module_with_submodules(x, y)
            > self.assertEqual(orig_out, submodules_out)
            True
    z%szpre split_moduleT)colorednodebase_mod_envbase_mod_attrsc                    s  | j dkrKt| jdkr| jd ntjj}r1|tjju rdn|f} jd| j|| jd|| j< n j	| j
| j|d|| j< | j || j _||fS | j dkr} | j
|| j< | j || j _t| j
tsrtdt| j
 t| j
}||| j
< ||fS )Nplaceholderr   r   )args	type_expr)r>   default_valueget_attrExpected str target, got )oplenr=   inspect	Signatureemptycreate_noder   typer<   targetmetacopyr@   
isinstancer$   AssertionErrorr.   )r9   r:   r;   r?   r=   r,   )base_mod_graphr6   r1   r   r   construct_graph   s6   


z%split_module.<locals>.construct_graphr   Ndef_nodeuse_nodec                    sR  ddl m} t| dd }t|dd }td| j||d ur|jnd| ||kr|d ur@ | }|j| j |d ur@|j| |d ur | }|j	| j | j
d }d urt||tdD ]5}| }	|j	|	j | jdkrt|	dd }
|
d ur |
 }|j|	j |j| |j|
 qa|d ur|j| d S d S d S d S )	Nr   )free_symbols_fx_partitionz*record_cross_partition_use %s (%s) %s (%s)-example_value)keyr<   )%torch.fx.experimental.symbolic_shapesrR   r+   logdebugr   r   
setdefaultr   r   rJ   getsortedr$   rB   r   )rP   rQ   rR   defineduseddef_partitionuse_partitiondef_valss_node	s_defineds_def_partition)
partitionssymbol_to_noder   r   record_cross_partition_use   sH   	z0split_module.<locals>.record_cross_partition_usec                    sp   | }t |} d urd |g}td| j| |}|d u r,t| |< }|j| j || _	d S )N_z*instantiate_node_partition_mapping %s (%s))
r$   joinrX   rY   r   r[   r
   r   appendrS   )r9   partition_idxpartition_name	partition)r/   rf   r3   r   r   "instantiate_node_partition_mapping
  s   

z8split_module.<locals>.instantiate_node_partition_mappingrU   )r<   r@   outputcall_function   z*Expected 1 arg for _set_grad_enabled, got zExpected bool arg, got c                 s   s    | ]	}t |t V  qd S N)rL   r	   .0argr   r   r   	<genexpr>Z  s    zsplit_module.<locals>.<genexpr>z3Expected all args to be python constants, not Nodesz'Expected 1 arg for _exit_autocast, got c                 s   s    | ]}|d uV  qd S rs   r   )ru   vr   r   r   rw   p      zautocast must exitc                 S      i | ]	\}}|t |qS r   r\   ru   krx   r   r   r   
<dictcomp>t      z split_module.<locals>.<dictcomp>c                 S   rz   r   r{   r|   r   r   r   r~   v  r   zautocast_regions: %szgrad_regions: %s)r<   r@   rp   c                    s
    | d S rs   r   n)rh   r   r   <lambda>     
 zsplit_module.<locals>.<lambda>zSautocast or set_grad_enabled require monotonically increasing partitions: highest: z, this node's: c                    
   |  S rs   r   rP   r9   rh   r   r   r     r   c                    r   rs   r   r   r   r   r   r     r   z cycle exists between partitions!z%Expected at least one region for nodec                 s   s    | ]}|V  qd S rs   r   rt   r   r   r   rw     s    )rB   rI   r=   kwargsr>   c                     s>   r} n	d  }  d7  j j|  jd}d < |S )Narg_rr   )r>   )r   r<   rH   )r   r<   )counterinpr7   
new_inputs
orig_nodesrn   r   r   add_placeholder  s   
z%split_module.<locals>.add_placeholderr@   rA   rS   c                        |  S rs   r   r   r   r   r   r         c                    r   rs   r   r   r   r   r   r     r   )call_moduler@   r'   ri   z&Expected tuple for gathered_args, got z'Expected dict for gathered_kwargs, got )rB   rI   r=   r   r>   r   zExpected at least one regionzMissing exit nodec                 S   s   g | ]	}|j d kr|qS )r<   )rB   )ru   r9   r   r   r   
<listcomp>Y  r   z split_module.<locals>.<listcomp>c                 3   s    | ]
}j  |  V  qd S rs   r   ru   r   )r   rn   r   r   rw   _  s    
c                    s   g | ]
}|vr | qS r   r   )ru   rV   )orig_mod_envoriginal_orderr   r   r   j  s
    c                 3   s    | ]} | V  qd S rs   r   r   r:   r   r   rw     ry   c                    s
    | j  S rs   )r   r   r   r   r   r     r   zpost split_module)OrX   rY   r   r	   dictr$   r   r   graph_moduler   sympyamp_enter_autocast_exit_autocast_C_set_grad_enabledr   setr   nodesrJ   r[   rL   SymIntSymFloatr9   exprSymbolrB   rI   rC   r=   rM   boolrH   alladdremovevaluesitems_LOGGERisEnabledForloggingDEBUGr   map_argr   listkeysr   rk   popr   RuntimeErrorr   rG   tuplerK   r   r.   nnModuler@   r   r)   rS   replacer   reversedr   r   rp   r   r   proxyProxy	enumeratenextiter)9r1   r2   r3   r4   r5   r6   r7   r/   r0   rO   r   ro   GLOBAL_STATE_NODESgrad_regionsautocast_regionsautocast_exitsactive_gradactive_autocastsvals0aassert_monotonically_increasinghighest_partitionpidoriginal_partition_orderroot_partitionsrm   sorted_partitionsroot_partition	dependentregions_mappingregionsrnew_node	orig_noder   	orig_attrr<   gathered_argsgathered_kwargsrI   target_attrr&   r   	exit_noder;   construct_order_partitionsalready_constructed_attr_nodesoutput_valsorig_mod_attr_nodes_based_mod_attrs
output_valnum_outputsoutput_val_proxyioutput_nameretr   )r:   rN   r   r   r   r7   r6   r1   r   r9   r   r   r   rn   r/   rf   rh   r3   rg   r   r   7   sN  t
#1
		


















	





)NFFT)!rD   r   collectionsr   collections.abcr   typingr   r   torch.fx._compatibilityr   torch.fx._lazy_graph_moduler   torch.fx._utilsr   torch.fx.graph_moduler   torch.fx.noder	   __all__	getLoggerr!   rX   r   r
   r   r   r$   r.   intr   r   r   r   r   r   r   <module>   sV   
		
