U Kcl @sRUddlZddlZddlZddlmZmZddlmZddlm Z m Z m Z m Z m Z mZddlZddlmmZddlmZddlmmZddlZddlmZddlmZmZddlm Z m!Z!ddl"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+d dl,m-Z-d dl.m/Z/m0Z0d dl1m2Z2zddl3m4Z5Wne6k rTddZ5YnXzddl7m8Z8Wne6k rddZ8YnXe9e j:dddde9e j;ddddejda?e dd d!Z@d"d#ZAd$d%ZBiZCgaDe eEeFd&<daGeHeFd'<d(aIeEeFd)<d*d+ZJee eEeEeHfd,d-d.ZKeEd,d/d0ZLeLZMedLd1d2ZNd3d4ZOd5d6ZPdMd7d8ZQejRGd9d:d:ZSe eeSd;dd?ZUe8e eeSd;d@dAZVGdBdCdCZWejeHeEeXeYgZZde2dddfe e e e e e e e eeHe dDdEdFZ[ej\ej\dGdHdIZ]ej\ej\dGdJdKZ^e[Z_e]Z`dS)NN)contextmanager nullcontextwraps)AnyCallableDictListOptionalTuple)Tensor)FakeTensorModeCrossRefFakeMode)immutable_collections Interpreter)ShapeEnv) stateless)make_fx) functionalize)enable_python_dispatcher)config)_named_buffers_named_parameters)default_partition)disablecCs|SNxrr?/tmp/pip-unpacked-wheel-gikjz4vx/functorch/_src/aot_autograd.pydisable_torchdynamosr!) dynamo_timedcCs|Srrrrrr r"&sr"cCs t|dfSr)listrrrr ,r$cCs t|Sr)rimmutable_listrcrrr r$-r%cCst|t|fSr)r#valueskeysrrrr r$1r%cCstddt||DS)NcSsi|]\}}||qSrr).0keyvaluerrr 3sz..)rimmutable_dictzipr'rrr r$2sc cs^ttj}tjr*ttj}z dVW5tj|tjrXtj|XdSr)torchclonerandomZ get_rng_stateZcudaZ is_availableZ set_rng_state)Z rng_stateZcuda_rng_staterrr preserve_rng_state:s    r4F)rootscsrdd}ddfdd}dd}||D]@}|jd g}||||}|d |||q,dS) Ncss|sdSt}t}|D] }|dk r||||q|r|}|jD].\}}||ksN|dkrhqN||||qN|Vq.iter_graphcsfdd}|S)NcstdadS)NF) fx_tracebackset_stack_trace callback_setr saved_stack_rr callbackas zKsetup_stacktrace_preservation_hooks..get_callback..callbackr)rErFrrDr get_callback`s z9setup_stacktrace_preservation_hooks..get_callbackcsfdd}|S)Ncs0ts"tjjjjtdat dS)NT) rCr1autogradvariableVariableZ_execution_engineZqueue_callbackrA format_stackrB) grad_output)rGstack_rr prehookis   zIsetup_stacktrace_preservation_hooks..get_prehook..prehookr)rMrNrG)rMr get_prehookhs z8setup_stacktrace_preservation_hooks..get_prehookcsfdd}|S)NcstdSr)rArB)Z grad_inputrLspecial_stack_rr posthookwszKsetup_stacktrace_preservation_hooks..get_posthook..posthookr)rRrSrrQr get_posthookvs z9setup_stacktrace_preservation_hooks..get_posthookZ traceback_zZforward_node_stackZ special_stackrrOr #setup_stacktrace_preservation_hooksKs  rYcs2tttttttttfdfdd }|S)N)primalstangentsreturnc s|}g}g}|D],}t|to&|j}|||r||qt|t|ksVtg}g}t||D],\} } t| trh| jrh|| || qhtdd|Dg} |rt t j j |||dd} W5QRXt | |fdd|DfS)NcSsg|] }|jqSr)Zgrad_fn)r+outrrr szQcreate_joint_forward_backward..joint_forward_backward..T)Z grad_outputsZ allow_unusedcsg|]}|rtndqSr)nextr+iZbackward_out_iterrr r^s) isinstancer requires_gradr:lenAssertionErrorr0rYrAoverride_stack_tracer1rHZgraditer) rZr[ZoutsZ grad_primalsZinputs_needs_gradspZis_grad_tensorZ needed_outsZneeded_tangentsr]ZtangentZ backward_outr?rbr joint_forward_backwards:      z=create_joint_forward_backward..joint_forward_backward)r rr )r?rkrrjr create_joint_forward_backwards (rlcCs&t|trt|St|tr |S|gSr)rctupler#rrrr normalize_as_lists   rngraph_being_compiled nth_graphmodel model_namecCs|adSr)rr)namerrr set_model_namesrt)r\cCsttttfSr)r#rorrrprrrr get_aot_compilation_contextsrucCstddtdtS)z7 Returns the name of the graph being compiled. _)rrjoinrorprrrr get_aot_graph_namesrxccs |gadV|rtd7agadS)Nr)rorp)Z graph_nameZincrement_indexrrr track_graph_compilings rycsfdd}d|_|S)Ncs|Srrargsfrr gszmake_boxed_func..gT) _boxed_call)r}r~rr|r make_boxed_funcs rcstfdd}|S)Ncs||}t|}|Sr)r)fx_gZinpsZout_fcompilerrr r}s zmake_boxed_compiler..fr)rr}rrr make_boxed_compilersrcCsL|s t|}t|tstt|dr2t||}ntdt||}|S)NraYour compiler for AOTAutograd is returning a a function that doesn't take boxed arguments. Please wrap it with functorch.compile.make_boxed_func or handle the boxed arguments yourself. See https://github.com/pytorch/pytorch/pull/83137#issuecomment-1211320670 for rationale.)r#rcrfhasattrrnwarningswarn)r}r{ steal_argsr]rrr call_func_with_argss  rc@s:eZdZUdZeed<eed<eed<eeefed<dS) AOTConfigz) Configuration for AOTDispatcher fw_compiler bw_compiler partition_fndecompositionsN)__name__ __module__ __qualname____doc__r__annotations__rrrrr rs r) flat_args aot_configc s^t||j|}tjr&td|td|||W5QRXtfdd}|S)Nz"====== Forward (only) graph ======Z inferencecst|}|Sr)r)r{fw_outs compiled_fwrr new_fns z!aot_dispatch_base..new_fn) rrr debug_graphsprintprint_readableryrr)flat_fnrr fw_modulerrrr aot_dispatch_bases rc si}g dit|d}t|D]N\}}||krP dd|||<q$ d|||<||<|d7}q$ fdd fdd |}t fd d } |} td d | } t| ttfrt| nd|| f} t j r0t || fd d } t t | j | j nt |j | t jrXtd trtd | \} W5QRXt jrtd| tdtd| |W5QRXW5QRXGfdddtjjtj fdd} | S)NFrTrcss|Sddt|DS)NcSsg|]\}}|r|qSrr)r+tZkeeprrr r^RszCaot_dispatch_autograd..remove_dupe_args..)r0rz) dropped_args keep_arg_maskrr remove_dupe_argsOsz/aot_dispatch_autograd..remove_dupe_argscs sSfddtDS)Ncsg|]}|qSrrr`) add_dupe_mapr{rr r^Wsz@aot_dispatch_autograd..add_dupe_args..)rangerz)rr duped_arg_lenrzr add_dupe_argsTsz,aot_dispatch_autograd..add_dupe_argscs |Srrrz)rrrr r$[r%z'aot_dispatch_autograd..cSst|tr|S|Sr)rcr detach contiguousrrrr r$_r%c s8tjj"tj||W5QRSQRXdSr)r1fx tracebackrgrrun)rZr[)rrr fake_fnpsz&aot_dispatch_autograd..fake_fnz====== Joint graph ======Zjointz====== Forward graph ======z====== Backward graph ======forwardcsDeZdZZdZZeefddZeefddZ dS)z/aot_dispatch_autograd..CompiledFunctionNcs4tj|}j}|j||dt|d|SNr)rrnum_outsZsave_for_backwardrm)ctxZdeduped_flat_tensor_argsrr)CompiledFunctionrr rsz7aot_dispatch_autograd..CompiledFunction.forwardc sndd|D}t|jt|}jdkrNtdd|_W5QRX|tj|dd}t|S)NcSsg|] }|qSr)r)r+rrrr r^szLaot_dispatch_autograd..CompiledFunction.backward..backwardT)r)r#Z saved_tensors compiled_bwryrZmaybe_clear_saved_tensorsrrm)rrZcontiguous_argsZall_argsr])rr bw_modulerr rs  z8aot_dispatch_autograd..CompiledFunction.backward) rrrrrr staticmethodr!rrr)r _num_outsrrcompiled_fw_funcrr rsrcsj|Sr)applyrz)rrrr compiled_functionsz0aot_dispatch_autograd..compiled_function)re enumerater:rlpytreeZtree_maprcr#rmrZuse_functionalizerrrgraphZeliminate_dead_codeZ recompileZ debug_jointrrr1Zno_gradryrrrrHZFunctionrr)rrrZ seen_argsjrarZdeduped_flat_argsrkr]Z joint_inputsrrrr) rrrrrrrrrrrrrr aot_dispatch_autograd$sj              rc sd|jdkri|_t|j|_tjr(dt_tjr4tnttjrFtnt}tjrXt nt}tjrjt ndt ||fdd}||}t dd|Dot }|rtt|||W5QRW5QRW5QRW5QRSt|||W5QRW5QRW5QRW5QRSW5QRXW5QRXW5QRXW5QRXdS)a8 Traces the forward and backward graphs of the attr:`flat_fn` to generate a joint graph. The joint graph is an Fx graph with Aten ops. Please refer to the tracing mechanism to understand the graph capturing details. The joint graph is then passed through attr:`partition_fn` to isolate the forward and backward portions, which are then respectively compiled via the provided attr:`fw_compiler` and attr:`bw_compiler`. The resulting compiled forward and backward graphs are then wrapped up in a ``torch.autograd.Function`` object. NFcs*tjr"fdd}tt||S|SdS)Ncsj|dS)N) shape_env)Z from_tensorrZ fake_moderrr convertszGcreate_aot_dispatcher_function..process_inputs..convert)ruse_fake_tensorrZ tree_map_onlyr )rrrrr process_inputssz6create_aot_dispatcher_function..process_inputscSsg|]}t|tr|jqSr)rcr rd)r+rrrr r^s z2create_aot_dispatcher_function..)raot_autograd_decompositionsrZdebug_fake_cross_refrr rrZuse_dynamic_shapesrrr4anyr1Zis_grad_enabledrrr)rrrZ cross_refZpython_dispatcher_moderZfake_flat_tensor_argsZneeds_autogradrrr create_aot_dispatcher_functions4    4rc@s(eZdZdZdZdZddZddZdS) PytreeThunkNcCsb|jdks|j|kst||_t|jttfkrJtdd|jDrJd|_t|jt j r^d|_ dS)Ncss|]}t|tjVqdSr)rcrLeafSpecr`rrr sz"PytreeThunk.set..T) specrftypermr#allZchildren_specs is_simplercrris_really_simple)selfrrrr r6 szPytreeThunk.setcCs&|jr|dS|jr|St||jSr)rrrtree_unflattenr)rrrrr unflattens zPytreeThunk.unflatten)rrrrrrr6rrrrr rs  r)r?rrrrstatic_argnumsr\csL|dk rtd|dkr|}t||||ddtfdd}|S)a Traces the forward and backward graph of :attr:`fn` using torch dispatch mechanism, and then compiles the generated forward and backward graphs through :attr:`fw_compiler` and :attr:`bw_compiler`. :func:`aot_function` traces the forward and backward graph ahead of time, and generates a joint forward and backward graph. :attr:`partition_fn` is then used to separate out forward and backward graphs. The partitioner function can be used to perform optimizations such as recomputation. One can set `decompositions` dictionary to decompose the operators into a sequence of core or simpler operators supported by the backend compilers. :func:`aot_function` uses a compilation cache, based on input tensor properties, to detect when there is a need of recompilation. .. warning:: This API is experimental and likely to change. Args: fn (Callable): A Python function that takes one ore more arguments. Must return one or more Tensors. fw_compiler (Callable): A Python function that accepts an Fx graph with Aten ops and input args, and returns a Callable that semantically is equivalent to the input Fx graph. bw_compiler (Optional[Callable]): A Python function that accepts an Fx graph with Aten ops and input args, and returns a Callable that semantically is equivalent to the input Fx graph. Default: None (when None, it defaults to the :attr:`fw_compiler`) partition_fn (Callable): A Python function that takes a joint forward and backward graph, and partitions it into separate forward and backward graphs. decompositions (Dict): A dictionary to define the decomposition of larger Aten ops into simpler or core Aten ops. Returns: Returns a ``Callable`` that retains the eager behavior of the original :attr:`fn`, but with forward and backward graph compiled via :attr:`fw_compile` and :attr:`bw_compile`. A simple example usage of :func:`aot_function` is as follows. This example will print the forward and backward graphs of the function ``fn`` >>> fn = lambda x : x.sin().cos() >>> def print_compile_fn(fx_module, args): >>> print(fx_module) >>> return fx_module >>> aot_fn = aot_function(fn, print_compile_fn) >>> x = torch.randn(4, 5, requires_grad=True) >>> aot_fn(x) NzTstatic_argnums has been deprecated - manually wrap your function or use torchdynamo.rrrrcspt||f\}}dkrVt||f\}tfdd}t||}|f\}||}|S)Nc sxt|\}}||}t|\}}|D]<}d}tD]}t||r8d}qPq8|s,tdt|dq,||S)NFTzFound aA in output, which is not a known type. If this type holds tensors, you need to register a pytree for it. See https://github.com/pytorch/functorch/issues/475 for a brief explanation why. If you don't need to register a pytree, please leave a comment explaining your use case and we'll make this more ergonomic to deal with)rr tree_flatten KNOWN_TYPESrc RuntimeErrorrr6) rr{kwargsZtree_outZflat_outrraZ is_known_typer)r?out_spectensor_args_specrr rts$   z8aot_function..returned_function..flat_fn)rrrrr)r{rrrvr compiled_fnZ cached_fnr]rZ cached_resr?)rrr returned_functionhsz'aot_function..returned_function)rrr)r?rrrr hasher_typerrrrr aot_function s;2r)modr\cs<fdd}t|f||GfdddtjS)a Traces the forward and backward graph of :attr:`mod` using torch dispatch tracing mechanism. It is wrapper function, that underneath uses :func:`aot_function` to perform tracing and compilation. :func:`aot_module` lifts the parameters and buffers of ``nn.Module`` as inputs to a new callable which is then compiled through :func:`aot_function`. .. warning:: This API is experimental and likely to change. Args: mod (Callable): A ``nn.Module`` module. args : args to be passed to :func:`aot_function` kwargs : kwargs to be passed to :func:`aot_function` Returns: Returns a ``nn.Module`` that retains the eager behavior of the original :attr:`mod`, but with forward and backward graph compiled. cs||}t|||Sr)rfunctional_call)Z named_paramsZ named_buffersr{rZparams_and_buffers)rrr rsz#aot_module..functional_callcs.eZdZfddZfddZZS)zaot_module..AOTModulecst||_dSr)super__init__Z orig_module)r) AOTModule __class__rrr rsz&aot_module..AOTModule.__init__cs*ttddttddf||S)NFZremove_duplicate)dictrr)rr{r) compiled_frrr rsz%aot_module..AOTModule.forward)rrrrr __classcell__rrrr)rr rsr)rnnModule)rr{rrrrr aot_modules  rcsttddttdd}t|\ttfdd}dtdddfttt ttt t tddd}||f|||rfd d }nfd d }j |_ j |_ |S) a This is the simplified or low overhead version of aot_module. For frontends like TorchDynamo, the input functions/modules to AOT are static and have unpacked inputs/outputs. This gives us an opportunity to remove the (1) pytree overhead to parse inputs/outputs, (2) AOT Autograd cache, (3) Reading of params/buffers in every forward call :func:`aot_module_simplified` removes these overheads. Frc stt|dttjjrt Xt Dt ddtj jddtj|d|}W5QRXW5QRXW5QRXn|d|}W5QRXt|ttfstd|S)Nignorez#Anomaly Detection has been enabled.F)Z check_nanzGraph output must be a tuple(). This is so that we can avoid pytree processing of the ouputs. Please change the module to have tuple outputs or use aot_module instead.)rZ_reparametrize_modulerrrcr1rZ GraphModulerArgrcatch_warningsfilterwarningsrHZdetect_anomalyrrrmr#r)r{rr])r params_len params_specrr rs$8z.aot_module_simplified..functional_callN)r?rrrrr\csH|dks t|dkr|}t||||ddtfdd}|S)Nrcsdkrt||Sr)rrzrrr?rr new_func szHaot_module_simplified..aot_function_simplified..new_func)rfrr)r?rrrrrrrrrr aot_function_simplifieds  z6aot_module_simplified..aot_function_simplifiedcs||Srr)r{rr params_flatrr rs z&aot_module_simplified..forwardcs |Srrrzrrr r%s) rrrrrrmrerrr rZ zero_gradZnamed_parameters)rZtop_argsZ top_kwargsparamsrrrr)rrrrrr aot_module_simplifieds6 " r)F)F)ar7Z dataclassesr contextlibrr functoolsrtypingrrrr r r r1Ztorch.fx.tracebackrrrAZtorch.nnrZtorch.utils._pytreeutilsZ_pytreerZtorch.utils.dlpackr Ztorch._subclassesr rZtorch.fxrrZ%torch.fx.experimental.symbolic_shapesrZtorch.nn.utilsrZ functorchrZfunctorch.experimentalrZtorch._dispatch.pythonrrZnamed_members_polyfillrrZ partitionersrZ torchdynamorr! ImportErrorZtorchdynamo.utilsr"Z_register_pytree_noder&r/opsZatenr4rCrYrlrnrrostrrrpintrrrtrurxZget_graph_being_compiledryrrrZ dataclassrrrrrfloatboolrrrrrrZcompiled_modulerrrr s            <,     L  ~-e