U
    Jcj                     @   s  U d dl Z d dlZd dlmZmZmZmZmZmZm	Z	 d dl
Z
d dlm  mZ d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dl mZmZ d dlZd dlmZ d dlZd dlZd d	l m!Z!m"Z"m#Z# d d
l$m%Z% ddl&m'Z'm(Z(m)Z)m*Z* d dlm+Z+ ddddddgZ,e
j-j.Z.e
j-j/Z/i a0ee
j1j2ef e3d< dZ4e5e
j6dd dd  dd Z7edd Z8e9 Z:e9 Z;dd Z<dd Z=e;d d fd!d"Z>d#d$ Z?d%d Z@d&d ZAd'd( ZBd)d* ZCd+d, ZDd-d. ZEd/d0 ZFeG d1d2 d2ZGd3d4 ZHd5d6 ZIe
jJe
jKjLfZMd7d8 ZNG d9d deZOdPe	e
jKjPef eeeed:f  ed;d<dZQd=d> ZRG d?d@ d@e!ZSe
jTZUe
jVZWG dAdB dBe(ZXG dCd de
jjYZZdDdE Z[edFdG Z\dQdIdZ]dJdK Z^e jdLdM Z_dRdNdOZ`dS )S    N)AnyCallableDictListOptionalTupleUnion)TracerGraphModule)FakeTensorMode)enable_python_dispatcher)_extract_tensor_metadata)contextmanagernullcontext)	dataclass)TorchDispatchMode_pop_mode_temporarily_get_current_dispatch_mode)
FakeTensor   )ShapeEnvSymDispatchModePySymInt
PySymFloat)ProxyPythonKeyTracerdispatch_tracemake_fxDecompositionInterpreter	get_proxy	has_proxyCURRENT_DECOMPOSITION_TABLEc                 C   s   t | d fS N)listx r&   F/tmp/pip-unpacked-wheel-gikjz4vx/torch/fx/experimental/proxy_tensor.py<lambda>%       r(   c                 C   s   t | S r"   )tuple)xs_r&   r&   r'   r(   %   r)   c                 C   s4   d dd t|D }td| d| dd| iS )z*FX gets confused by varargs, de-confuse it,c                 s   s   | ]}d | V  qdS )argNr&   ).0ir&   r&   r'   	<genexpr>)   s     z!fake_signature.<locals>.<genexpr>zlambda z: fn()fn)joinrangeeval)r3   nargsargnamesr&   r&   r'   fake_signature'   s    r9   c                 c   s   t }| a z
t V  W 5 |a X d S r"   )r!   )decomposition_tableZold_decomposition_tabler&   r&   r'   	decompose,   s
    
r;   c                 C   s.   | j tt }t|tjs"t|||< d S r"   )__dict__
setdefault
proxy_slotweakrefWeakKeyDictionary
isinstanceAssertionError)objtracerproxydr&   r&   r'   set_proxy_slot:   s    rG   c                 C   s   t | |ddd S )NFc                 S   s   dS NTr&   r,   r&   r&   r'   r(   @   r)   z has_proxy_slot.<locals>.<lambda>get_proxy_slot)rC   rD   r&   r&   r'   has_proxy_slot?   s    rL   c                 C   s   | S r"   r&   r$   r&   r&   r'   r(   E   r)   c                 C   sr   | j t}|s.|tkr*t|  d| |S t|tjs>t||krf|tkrbt|  d| n|S ||| S )Nz is not tracked with proxy for )	r<   getr>   
no_defaultKeyErrorrA   r?   r@   rB   )rC   rD   defaultZ	transformrF   r&   r&   r'   rK   E   s    rK   c                 C   s   | j tS r"   )r<   rM   r>   rC   r&   r&   r'   get_proxy_slotsT   s    rR   c                 C   s8   t | }|d krd S t| }t|dks0t|d S )Nr   r   )rR   r*   valueslenrB   )rC   resvalsr&   r&   r'   r   Y   s    c                 C   s   t | d k	S r"   )r   rQ   r&   r&   r'   r    a   s    c                 C   sf   t |tr(|| jjd< t|| jjd< n:t |tr@|| jjd< n"t |tjrb|jsbt|| jjd< | S )NvalZtensor_meta)	rA   r   nodemetar   r   torchTensorZ	is_sparse)rE   rW   r&   r&   r'   set_metad   s    

r\   c                 O   s   t dt j| f||S )zT
    Delays computation of f until it's called again
    Also caches the result
    r   )	functools	lru_cachepartial)fargskwargsr&   r&   r'   thunkifyo   s    rc   c                   s   fdd}t | jD ]\}}|| fdd| qt |  D ]\}}|| fdd| q@||   fdd ||   fdd t| t | d S )Nc                    sH   t |stt| trD|  }t|ts,tt| t||f|  d S r"   )callablerB   rA   SymInt	get_pyobjr   rG   rc   )Zouter_sZproxy_callablera   Zinner_srD   r&   r'   try_set_proxy_slotw   s
    
z(track_tensor.<locals>.try_set_proxy_slotc                    s   t tjj || S r"   )r\   rZ   opsatenZsym_sizer%   r0   rE   r&   r'   r(      r)   ztrack_tensor.<locals>.<lambda>c                    s   t tjj || S r"   )r\   rZ   ri   rj   Z
sym_striderk   rl   r&   r'   r(      r)   c                    s   t tjj | S r"   )r\   rZ   ri   rj   Z	sym_numelr$   rl   r&   r'   r(      r)   c                    s   t tjj | S r"   )r\   rZ   ri   rj   Zsym_storage_offsetr$   rl   r&   r'   r(      r)   )	enumerateshapeZstridenumelZstorage_offsetrG   _ProxyTensor)ZtensorrE   constantrD   rh   r0   sr&   )rE   rD   r'   track_tensorv   s    rs   c                   st   fdd fddt | ts0t | trXt| D ]\}}||| | q8nt | tjrp| |  | S )Nc                    s\   t | tjr(t| ||d t||  n0t | trXt| D ]\}}|||  | q:d S )N)rD   rq   )rA   rZ   r[   rs   r\   r#   rm   )erE   rq   idxee)get_constantrD   wrap_with_proxyr&   r'   rx      s    
z*track_tensor_tree.<locals>.wrap_with_proxyc                    s    d krd S  |  S d S r"   r&   )ru   rq   r&   r'   rw      s    z'track_tensor_tree.<locals>.get_constant)rA   r*   r#   rm   rZ   r[   )Z	inner_resZ	proxy_resrq   rD   ru   rt   r&   )rq   rw   rD   rx   r'   track_tensor_tree   s    		rz   c                  C   s    t  } t| trt S t S d S r"   )r   rA   r   r   r   )Zmb_fake_moder&   r&   r'   maybe_disable_fake_tensor_mode   s    
r{   c                   @   s$   e Zd ZU eed< eej ed< dS )rp   rE   rq   N)__name__
__module____qualname__r   __annotations__r   rZ   r[   r&   r&   r&   r'   rp      s   
rp   c                    s    fdd}|S )Nc                    s(   |   }|jd k	r|jS t|  S d S r"   )rf   rq   rK   )rt   nrg   r&   r'   inner   s    
zfetch_sym_proxy.<locals>.innerr&   )rD   r   r&   rg   r'   fetch_sym_proxy   s    r   c                    s    fddS )Nc                    s   t |  | S r"   rJ   trg   r&   r'   r(      r)   z$fetch_tensor_proxy.<locals>.<lambda>r&   rg   r&   rg   r'   fetch_tensor_proxy   s    r   c              
      s    fdd}t tj|||fs$tS |tkrb , t| ||}|tk	rX|W  5 Q R  S W 5 Q R X  * |j||}|tk	r|W  5 Q R  S W 5 Q R X  j}t tjt	|||f\}}t t
dd ||fot ttfdd ||f}	tjj|jkrD|	r<t t
dd ||f\}
}t  ||
|W  5 Q R  S Q R X tdt ttft jt t
dd ||f\}}|tjjjjkrtjjjj} jjd	||| jj|jjd
}|jjd dkr|jjd dkrt|d trt|d D ]\}}|d | |_ qn
||d _ |||}t !t
dd ||f}d }|tjjjjkr~|" t#kr~t  |d $ }W 5 Q R X nntjj%|jkr|	r|rt tjdd |rt * t t
dd ||f\}
}||
|}W 5 Q R X nd }t&||||d |S )Nc                    s   t | tkpt|  jS r"   )typeHANDLED_TYPESrL   rD   r$   
proxy_moder&   r'   can_handle_tensor   s    z%proxy_call.<locals>.can_handle_tensorc                 S   s
   | j d k	S r"   ry   r   r&   r&   r'   r(      r)   zproxy_call.<locals>.<lambda>c                 S   s   dS NFr&   rI   r&   r&   r'   r(      r)   c                 S   s   | j S r"   ry   r   r&   r&   r'   r(      r)   zIt appears that you're trying to get value out of a tracing tensor - erroring out! It's likely that this is caused by data-dependent control flow or similar.c                 S   s   | j S r"   rl   rt   r&   r&   r'   r(      r)   call_function)namer,   r   c                 S   s
   | j d k	S r"   ry   r   r&   r&   r'   r(   G  r)   c                 S   s   |   tkS r"   )ro   CONSTANT_NUMEL_LIMITr   r&   r&   r'   r(   U  r)   c                 S   s   | j S r"   ry   r   r&   r&   r'   r(   Z  r)   rq   rD   )'pytreeZtree_all_onlyrZ   r[   NotImplementedr!   r;   rD   tree_map_onlyr   rp   re   SymFloatTagZdata_dependent_outputtagsr{   RuntimeErrorr   ri   rj   Z
lift_freshrP   Zlift_fresh_copyZcreate_proxygraphZ_target_to_strZoverloadpacketr|   rA   r   rm   rE   Ztree_any_onlyro   r   cloneZnondeterministic_seededrz   )r   funcra   rb   r   rrD   Zf_argsZf_kwargsZall_constantZ
const_argsZconst_kwargsZ
proxy_argsproxy_kwargsZ	proxy_outr0   aoutZany_constantrq   r&   r   r'   
proxy_call   s      '$

   r   c                       sh   e Zd Z fddZejjedef e	edf e
eef edddZdd Zed	 fd
dZ  ZS )r   c                    s   t    d S r"   )super__init__self	__class__r&   r'   r   e  s    zPythonKeyTracer.__init__.)mforwardra   rb   returnc                 C   s
   |||S r"   r&   )r   r   r   ra   rb   r&   r&   r'   call_modulek  s    zPythonKeyTracer.call_modulec                 C   s   |S r"   r&   )r   attrZattr_valZparameter_proxy_cacher&   r&   r'   getattrq  s    zPythonKeyTracer.getattr)r   c                    s   t |tjjr| j D ]$\}}||kr| d|di   S qd }|szd}d| }t| j|sbql|d7 }qJt| j|| | d|di S t |t	t
fr| jd k	st| jS t |S )Nget_attrr&   r   Z_param_constantr   )rA   rZ   nn	ParameterrootZnamed_parameterscreate_nodehasattrsetattrre   r   rf   rq   rB   r   
create_arg)r   r   r   pqualnamer0   r   r&   r'   r   t  s"    


zPythonKeyTracer.create_arg)r|   r}   r~   r   rZ   r   Moduler   r   r   r   strr   r   r   __classcell__r&   r&   r   r'   r   d  s    
 
 
.)r   rD   concrete_argsr   c                 C   s6   | | |}t| tjjr"| jjn| j}t|j||S r"   )	tracerA   rZ   r   r   r   r|   r
   r   )r   rD   r   r   r   r&   r&   r'   r     s    c                    s.   t \}t  fdd}|S )Nc                     sp   t | \}}t|tks"tt|d d   }t tjfdd|}t tt	ffdd|}|S )Nr   c                    s   t |  | dd S )Nc                 S   s   | j S r"   rl   r$   r&   r&   r'   r(     r)   z=wrap_key.<locals>.wrapped.<locals>.<lambda>.<locals>.<lambda>rJ   r   rg   r&   r'   r(     r)   z+wrap_key.<locals>.wrapped.<locals>.<lambda>c                    s   t |    S r"   )rK   rf   r   rg   r&   r'   r(     r)   )
r   tree_flattenrT   rB   rz   r   rZ   r[   re   r   )proxiesZflat_proxiesZproxies_specr   r`   Zflat_tensorstensorsrD   r&   r'   wrapped  s    

zwrap_key.<locals>.wrapped)r   r   r]   wraps)r`   r   rD   Ztensors_specr   r&   r   r'   wrap_key  s    r   c                       sD   e Zd Zdd ZdddZ fddZ fd	d
ZdddZ  ZS )ProxyTorchDispatchModec                 C   s&   || _ d| _t|| _i | _g | _d S rH   )rD   enable_tracingProxySymDispatchModesym_modeZtrace_state	_managersr   rD   r&   r&   r'   r     s
    
zProxyTorchDispatchMode.__init__r&   Nc              
   C   s4   | j d | ||||W  5 Q R  S Q R X d S r   )r   enableinner_torch_dispatch)r   r   typesra   rb   r&   r&   r'   __torch_dispatch__  s    z)ProxyTorchDispatchMode.__torch_dispatch__c                    s*   | j d}| j| |  t  S rH   )r   r   r   append	__enter__r   )r   r   r   r&   r'   r     s    z ProxyTorchDispatchMode.__enter__c                    s>   | j  }t |||}|s,||||S |d d d S d S r"   )r   popr   __exit__)r   exc_type	exc_value	tracebackr   br   r&   r'   r     s
    
zProxyTorchDispatchMode.__exit__c                 C   s:   | j s|||S |tjjfkr(|||S t| |||}|S r"   )r   primZdevicerP   r   )r   r   r   ra   rb   r   r&   r&   r'   r     s    

z+ProxyTorchDispatchMode.inner_torch_dispatch)r&   N)r&   N)	r|   r}   r~   r   r   r   r   r   r   r&   r&   r   r'   r     s
   
	r   c                       s8   e Zd Z fddZedd Zdd Zdd Z  ZS )	r   c                    s   t    || _d| _d S rH   )r   r   rD   r   r   r   r&   r'   r     s    
zProxySymDispatchMode.__init__c                 c   s$   | j }|| _ z
d V  W 5 || _ X d S r"   r   )r   r   oldr&   r&   r'   r     s
    
zProxySymDispatchMode.enablec                    sD   t  fdd|D } jd||i }t| j}t|| |S )Nc                 3   s@   | ]8}|j d kr t| j jnt|ttfr4|j n|V  qd S r"   )rq   rK   rD   rX   rA   r   r   )r/   r   r   r&   r'   r1     s
   z6ProxySymDispatchMode._compute_proxy.<locals>.<genexpr>r   )r*   rD   r   fxr   r\   )r   r   ra   r   Zn_argsZn_outZp_outr&   r   r'   _compute_proxy  s    
z#ProxySymDispatchMode._compute_proxyc                 C   s   | j s|||S |tjkrbt|d tr>|d jdkr>|d S t|d trb|d jdkrb|d S |rjt|||}t|ttfst| d| d| d| t| j	|||d}t
|| j| |S )Nr   r   z(*z, **z) = )r   ra   r   )r   operatormulrA   r   rq   rB   r   rc   r   rG   rD   )r   r   r   ra   rb   r   Zp_out_thunkr&   r&   r'   __sym_dispatch__  s    


,z%ProxySymDispatchMode.__sym_dispatch__)	r|   r}   r~   r   r   r   r   r   r   r&   r&   r   r'   r     s
   
r   c                       s^   e Zd Zdejjejjd fddZ fddZ fddZ	 fd	d
Z
 fddZ  ZS )r   N)module	new_graphc                    sN   t  j|f| || _tjj| j| _|| _| jd kr>i | _t	| j| _
d S r"   )r   r   r   rZ   r   rE   ZGraphAppendingTracerrD   r:   r   mode)r   r   r   r:   rb   r   r&   r'   r     s    
z!DecompositionInterpreter.__init__c                    s>   t  |||}tj| j|| j}t||d | jd |S Nr   )r   placeholderrZ   r   r   r   rD   rz   r   targetra   rb   r   rE   r   r&   r'   r     s    z$DecompositionInterpreter.placeholderc                    s>   t  |||}tj| j|| j}t||d | jd |S r   )r   r   rZ   r   r   r   rD   rz   r   r   r&   r'   r   $  s    z!DecompositionInterpreter.get_attrc                    s4   t  |||} fdd} jt|| |S )Nc                    s   t |  j| dd S )Nc                 S   s   | j jS r"   )rE   rX   r$   r&   r&   r'   r(   0  r)   zADecompositionInterpreter.output.<locals>.unwrap.<locals>.<lambda>)rK   rD   r   r   r&   r'   unwrap/  s    z/DecompositionInterpreter.output.<locals>.unwrap)r   outputr   r   tree_map)r   r   ra   rb   r   r   r   r   r'   r   ,  s    zDecompositionInterpreter.outputc                    sN   t | j: | j( t j||W  5 Q R  W  5 Q R  S Q R X W 5 Q R X d S r"   )r;   r:   r   r   run)r   ra   rb   r   r&   r'   r   4  s    zDecompositionInterpreter.run)N)r|   r}   r~   rZ   r   r
   ZGraphr   r   r   r   r   r   r&   r&   r   r'   r     s
   	c                    s(   t ||f\} fdd}||fS )Nc                    s   t | \}} ||S r"   )r   Ztree_unflatten)	flat_argsZfn_argsZ	fn_kwargsr   specr&   r'   r   @  s    z-wrapper_and_args_for_make_fx.<locals>.wrapped)r   r   )r   ra   rb   r   r   r&   r   r'   wrapper_and_args_for_make_fx;  s    r   c               	   c   s.   t  } t d z
d V  W 5 t |  X d S r   )rZ   Zis_autocast_cache_enabledZset_autocast_cache_enabled)	old_valuer&   r&   r'   disable_autocast_cacheE  s
    

r   realc                    s6   dkst  d kri  t fdd}|S )Nr   fakesymbolicc                     s  t dd | }t }t  dkr,t  n6dkr@tdd n"dkrTtdd ntd	 t }dkrvt }t|} fd
d}d dkrt |j	} fdd}dd ||d}t | | } t
tdrtjjtj@ rtt|}	n}	tr  b |R |B |2 t   tt|	| ||t|d}
W 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X dkr|
_|
S )Nc                 S   s   t jS r"   )r   ZPHrI   r&   r&   r'   r(   W  r)   z*make_fx.<locals>.wrapped.<locals>.<lambda>r   r   T)Zallow_fallback_kernelsr   FzUnexpected tracing type: c                    s   t | tjr | S | S r"   rA   rZ   r[   Zfrom_tensorr$   )fake_tensor_moder&   r'   wrap_fake_concretei  s    
z4make_fx.<locals>.wrapped.<locals>.wrap_fake_concretec                    s   t | tjr j| dS | S )N)	shape_envr   r$   r   r   r&   r'   wrap_fake_symbolicu  s    z4make_fx.<locals>.wrapped.<locals>.wrap_fake_symbolicc                 S   s   | S r"   r&   r$   r&   r&   r'   r(   {  r)   r   __code__)rD   r   )r   r   r   r   r   rB   r   r   r   r   r   inspectr   r   co_flags
CO_VARARGSr9   rT   r;   r   r   r   r*   r   )ra   ZphsZ	fx_tracerZpython_dispatcher_moder   r   r   r   Zwrap_fn_mapr   r   r:   r`   tracing_moder   r'   r   U  sN    &V
zmake_fx.<locals>.wrapped)rB   r]   r   )r`   r:   r   r   r&   r   r'   r   O  s    =c                   C   s   t jj S r"   )rZ   utilsZ_python_dispatchZ _get_current_dispatch_mode_stackr&   r&   r&   r'   get_torch_dispatch_modes  s    r   c               	   c   s^   t  } dd | D }dd |D }|D ]
}d|_q&z
d V  W 5 t||D ]\}}||_qHX d S )Nc                 S   s   g | ]}t |tr|qS r&   )rA   r   r/   r   r&   r&   r'   
<listcomp>  s     
 z/disable_proxy_modes_tracing.<locals>.<listcomp>c                 S   s   g | ]
}|j qS r&   r   r   r&   r&   r'   r    s     F)r   r   zip)modesZproxy_tensor_modesZoldsr   r   r&   r&   r'   disable_proxy_modes_tracing  s    
r  c              	   C   s6   t | ||\}}t  t||d|}W 5 Q R X |S )a  A helper function used to get the GraphModule for the given func.

    It's expected to be used in the ProxyTensor tracing context.
    It detaches the args and kwargs from the current tracer so that the trace of
    the current graph module can be created without any side-effects.
    )r   )r   r  r   )r   ra   rb   r   r   Zall_argsZgmr&   r&   r'   get_isolated_graphmodule  s    r  )N)Nr   )r   )a
contextlibr]   typingr   r   r   r   r   r   r   rZ   Ztorch.utils._pytreer   Z_pytreer   Ztorch.fxr	   r
   Ztorch._subclasses.fake_tensorr   Ztorch._dispatch.pythonr   r   Ztorch.fx.passes.shape_propr   r   r   r   Zdataclassesr   r?   r   Ztorch.utils._python_dispatchr   r   r   Ztorch._subclassesr   Zsymbolic_shapesr   r   r   r   r   __all__ri   rj   r   r!   Z_opsZ
OpOverloadr   r   Z_register_pytree_nodeSizer9   r;   objectr>   rN   rG   rL   rK   rR   r   r    r\   rc   rs   rz   r{   rp   r   r   r[   r   r   r   r   r   r   r   r   r   Z
SymIntNodere   ZSymFloatNoder   r   ZInterpreterr   r   r   r   r   r  r  r&   r&   r&   r'   <module>   s    $


 * 
':(

	
G
