U <c@s6ddlZddlZddlZddlZddlZddlmZmZmZddl m Z ddl m Z m Z mZmZmZmZmZmZmZddlZddlmZddlmmZddlmmZddlmmZddlmm Z ddl!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.dd l/m0Z0m1Z1m2Z2m3Z3dd l4m5Z5m6Z6m7Z7m8Z8dd l9m:Z:dd l;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRdd lSmTZTmUZUmVZVmWZWddlXmYZYddlZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdedZeGdddedZfiagiahedddgZidejeekeeeAekeleidddZmdejekeekdddZnejoddekeekdd d!Zpddd"ekekeeeAeleid#d$d%ZqeeGdd&d'd(Zrekekd)d*d+Zsee=eeAd,d-d.ZteGe=eeAd/d0d1ZueGee=eekd/d2d3Zvee=eekd,d4d5Zwee)e+fe)ekd6d7d8Zxee)e+feGe=ekd9d:d;Zyee)e+feGee=ekd<d=d>Zzee)e+feGee=ekd<d?d@Z{e dAdBGdCdDdDZ|e dAdBGdEdFdFZ}e dAdBGdGdHdHZ~e dAdBGdIdJdJZe dAdBGdKdLdLZe2eGekdMdNdOZeHeekdPdQdRZeGeYeldSdTdUZe dAdBGdVdWdWZejekdXdYdZZekejd)d[d\ZePekd]d^d_ZeeQeekd`dadbZeGeee ekekfe ekekffdMdcddZe(eleekeeke ekekfejdedfdgZeF)es valid_tags ignore_keyspathskip_native_fns_genreturnc st|tstg}tt}|D]|}t|dts:t|t||d|dtfdd0t |||\}} | |t || W5QRXqt|tdd} |st|||D]"\} } t | ddt| | d| | <qt|| S) NrMfunccsddSNzin z: rVrVfuncslocrVrWz*parse_native_yaml_struct..cSsttjdddidS)NTF dispatch_keyZuse_out_as_primaryZexternal device_guardindex)r$r( UndefinedrVrVrVrWrks TFrm) isinstancelistAssertionErrorrdictgetintr-rAr.Z from_yamlappendr$Z grow_indexerror_check_native_functionsr:itemsr*r\) r`rarbrcrdrsbserfmindiceskvrVrhrWparse_native_yaml_structs4     r)r`rcrec st|tstt}|D]}t|dts4t|t||d|dtfdd:|}| d}| dd}|dkst| |W5QRXq|S)NrMtagcsddSrgrVrVrjtagsrVrWrkrlz(parse_tags_yaml_struct..desc) rrrsrtsetrvrwr-rAcopypopadd)r`rcr{r}Ze_inamerrVrrWparse_tags_yaml_structs    r)maxsize)rcrec CsD|tkr-sz(static_dispatch_keys..)rr(CompositeImplicitAutograd%CompositeImplicitAutogradNestedTensorCompositeExplicitAutograd&CompositeExplicitAutogradNonFunctionalrrVrVrWstatic_dispatch_keys)s  r)r backend_indexrecCsN|jdk s||r|jS|jr&tjS|jr2tjS|jr>tj S|j rJtj SdSN) r has_kernelrn&has_composite_explicit_autograd_kernelr(r5has_composite_explicit_autograd_non_functional_kernelr&has_composite_implicit_autograd_kernelr4has_composite_implicit_autograd_nested_tensor_kernelr)rrrVrVrWget_static_dispatch_backend5srcCsZ|dks|jrdSg}|D]4}t||}|dk r|d|jd|dqd|S)N#include r)manual_kernel_registrationrrx root_namelowerjoin)rroutputrprnrVrVrWstatic_dispatch_ops_headerIs rcCsddt|DS)NcSsg|]}d|dqS)#include rV)rrnrVrVrWrZsz1static_dispatch_extra_headers..)rrrVrVrWstatic_dispatch_extra_headersYsr)sigcpp_sigrecCsrttttddd}t|}t|}|D]}|jjtjkr4||}qTq4t||}d dd|DS)N)input_bindingsrecSsTg}|D]F}|jdkrDtttj|jj|j|j|jd}| |q| |q|S)NZ memory_format)nctyperdefaultargument) rrrr possibly_redundant_memory_formatrtyperrrx)rZoutput_bindingsZbindingZspl_mem_format_bindingrVrVrWadd_spl_memory_format_bindingis   z5translate_args..add_spl_memory_format_binding, css|] }|jVqdSrexprrarVrVrW sz!translate_args..) rrrs argumentsrrrrrr)rrrZ src_bindingsZ goal_bindingsargexprsrVrVrWtranslate_argscs   r)rrrrec Cstj|ddd}|jr(|jr(|j}n|j}|dk s:t|}t ||}| |}|rf|j rf|j nt }| dd} d| d|jd|d|d S) NFmethodZfallback_binding::nativerreturn ::();)rfrom_native_functionsymintrf has_symintsymint_signature signaturertrr get_kernel cpp_namespacer'rrnr) rrrcpp_sigsrrrbackend_metadataZ kernel_nsnsrVrVrW%generate_static_dispatch_backend_calls(    r)rrr^rec Cs(tj|ddd}|jr(|jr(|j}n|j}|dk s:t|}t ||}t dd}|j rd|dt jd|d|d S|jrd|dt jd|d|d S|jrd|dt jd|d|d S|jrd|dt jd|d|d Sd |d d d d |DdSdS)NFrrrrrrrz5TORCH_CHECK(false, "Static dispatch does not support z forrcSsg|]}t|jqSrV)rrn)rrprVrVrWrsz:generate_static_dispatch_fallback_call..z ");)rrrrfrrrrtrrr'rrr(rrrrrrrrr)rrr^rrrrrrVrVrW&generate_static_dispatch_fallback_calls.   $$$$rc sHt|dksjrdSfdd|D}t|dkrDt||dSt|dkr\t||Sdd|D}d|}jjj}g}g}|dk r|d |dkr|d |d |d d |d|dg} |D]2} | d| j d| dt|| dqt||} d} d| |d| | d| dS)a  For a given `NativeFunction`, find out the corresponding backend and dispatch to it. If more than one backends exsit, fallback to static dispatch by determining dispatch key from inputs. Arguments: sig: A CppSignature or DispatcherSignature for this native function we want to use. f: NativeFunction to generate static dispatch. backend_indices: All available backends. Return: C++ code to call backend-specific functions, e.g., "return at::cpu::add(self, other, scale);" rrcs.g|]&}|s&jdk r|jtkr|qSr)rrrnr5)rbrrVrWrs    z#static_dispatch..rLcSs6g|].}t|jts,t|jtr|jjr|jqSrV)rrrr4r#ris_tensor_likerrrVrVrWrs    rNz>DispatchKeySet(c10::computeDispatchKey(dtype, layout, device))z$c10::detail::multi_dispatch_key_set()zDispatchKeySet _dk_set = z | ;z=DispatchKey _dk = c10::highestPriorityBackendTypeId(_dk_set);zcase DispatchKey:::rz  z switch (_dk) { z default: z } ) rrrrrrrftensor_optionsrxrn) rrr^keysnative_tensor_args tensor_argsZ tensor_optsZstmtsZsubexprsZ dispatch_coderpfallbackZ connectorrVrrWstatic_dispatchsN        rT)frozenc@s.eZdZUeed<eeeedddZ dS)RegisterSchemaselectorrrecCsJ|j|sdSdddd|jDd}dtt|jd|dS)N{rcSsg|]}d|qS)z at::Tag::rV)rrrVrVrWrsz+RegisterSchema.__call__..}zm.def(z); )ris_native_function_selectedrrrrrf)rQrrrVrVrW__call__ s zRegisterSchema.__call__N) rXrYrZr>__annotations__rr.r rrrVrVrVrWr s rc@sJeZdZUeeejeejfed<e e ed<e e e dddZdS)ComputeOperatorstargetstatic_dispatch_backend_indicesrc Cst|j}|jj}|jtjkrd|d|d|jjjd|jjj dt t |jd|j ddd d |j d d d d S|jtj krd|d|jjjd|d|jjj d|dt t |jd|jd|d|d|d|d|d}dD]}|r(ddgdd|D}d }nddd|D}d}|}|d |} d!|d"|d#|d$} |st|jd%krt|||jd&} |d'|jd(|j | |d d)| d*7}q|St|jdS)+Nz struct TORCH_API z { using schema = z; using ptr_schema = schema*; // See Note [static constexpr char* members for windows NVCC] STATIC_CONSTEXPR_STR_INL_EXCEPT_WIN_CUDA(name, "aten::z>") STATIC_CONSTEXPR_STR_INL_EXCEPT_WIN_CUDA(overload_name, "z:") STATIC_CONSTEXPR_STR_INL_EXCEPT_WIN_CUDA(schema_str, z ) static callF)ris_redispatching_fnz ; static Z redispatchTz; };z+ STATIC_CONST_STR_OUT_OF_LINE_FOR_WIN_CUDA(z, name, "aten::z-") STATIC_CONST_STR_OUT_OF_LINE_FOR_WIN_CUDA(z, overload_name, "z, schema_str, z ) // aten::z. static C10_NOINLINE c10::TypedOperatorHandle create_zQ_typed_handle() { return c10::Dispatcher::singleton() .findSchemaOrThrow(z::name, z::overload_name) .typed(); } )FTrdispatchKeySetcSsg|] }|jqSrVrrrVrVrWrXsz-ComputeOperators.__call__..cSsg|] }|jqSrVrrrVrVrWr\srz static auto op = create_z_typed_handle(); return op.rrrr^ // aten::r { z } )r from_schemarfrunambiguous_namer rF DECLARATIONrrrrdefn DEFINITIONrrrr rr?) rQrrrZdefnsr Zdispatcher_exprs_strZ method_baseZdispatcher_call method_nameZfn_bodyrVrVrWrs            zComputeOperators.__call__N)rXrYrZrrrFrrrrr$rr.rrrVrVrVrWr s  r c@s$eZdZeeeedddZdS)ComputeFunctionrc Cstj|jkrdStj|d|jd}d}|D]f}t|j }t | | }d dd|D}|d|j d| d |j jd |d 7}q.|S) NFrrrcSsg|] }|jqSrVrrr}rVrVrWrsz,ComputeFunction.__call__..r inline  { return at::_ops::::call(); } )r8functionvariantsrrmanual_cpp_binding signaturesrrrfrrrdeclrrrQr sig_groupresultrZ target_sigrZ exprs_strrVrVrWr}s,     zComputeFunction.__call__NrXrYrZrr.r rrrVrVrVrWr{src@sNeZdZUeeejeejfed<e e ed<e e e edddZdS)ComputeTensorMethodr r rc Cs tj|jkrdS|jrt|jjjdk s0ttj |d|j d}|j t j krvd}|D]}||d7}qZ|S|j t jk rt|j d}|D]n}t|j}t||dd}ddd|D}|d |jd |jd d d |jjd|d 7}q|S)NTrrz const; )rrcSsg|] }|jqSrVrrrVrVrWrsz0ComputeTensorMethod.__call__..rrzTensor::)prefixz const { return at::_ops::rr)r8rr rf is_out_fnrtrZself_argrrr!r rFrr"r#rr?rrrrrrrr$rVrVrWrs>          zComputeTensorMethod.__call__N)rXrYrZrrrFrrrrr$rr.r rrrVrVrVrWr(s  r(c@s$eZdZeeeedddZdS)ComputeRedispatchFunctionrc Cstj|d|jd}d}|D]p}t|j}t||}d dgdd|D}|d|jd |j d d d |jj d |d 7}q|S)NFrrrrcSsg|] }|jqSrVrrrVrVrWrsz6ComputeRedispatchFunction.__call__..rrT)r rz ::redispatch(r) rrr!r"rrrfrrrr#rrr$rVrVrWrs(     z"ComputeRedispatchFunction.__call__Nr'rVrVrVrWr+sr+rcCsd|jjjd|jjjdS)Nz{"aten::z", "z"},)rfrrrrVrVrWcompute_aten_opsr,grecCs|js dSt|jt|}t|}ddd|D}|jj}|dkrTd}d}|jrf|jjnd}|rZ|j |j f}dd|D}dd|D} dd d| D} d | d } d d|D} d dd| D} g}t |D]"\}}d| d|dg| |dd}d|d }| |j dd}|d|jd|d}d||jd}d| |d|d}g}||dt |D]F\}}||kr|d|jdn|d|jd|jd q|d!d"|}|d#|d$|d%|d&qd"|}ddgt| }d'|d(}d)}d*| d+|d#| d,}nd-}d-}d.|d/|d0|d1|d1|d2|d3 W5QRSQRXdS)4Nrcss|]}|VqdSr)r#rrVrVrWrsz4compute_meta_function_declaration..zat::impl::MetaBasevoidcSsg|]}|D]}|q qSrVrV)rZ replace_listelemrVrVrWrsz5compute_meta_function_declaration..cSsg|]}|jqSrV)rupperrr0rVrVrWrscss|]}d|dVqdS)zbool z = falseNrV)rparamrVrVrWrsz template <>cSsg|]}tj||jdqS))binds)r argument_typerr2rVrVrWr sz; css&|]}|jddd|jVqdS)TZ strip_ref N)cpp_typerr2rVrVrWrstruerLzprecompute_outrz return ret;rz z { z z' } z'using meta_return_ty = precompute_out ;Zmeta_return_tyz zG struct TORCH_API precompute_out { z; };rzstruct TORCH_API structured_z : public rrz meta(z); }; )rroutmetarZmeta_argumentsrZstructured_inherits precomputedrvaluesr enumerater9rxr)r.rargsargs_strZ parent_classZ meta_returnr=Zprecomputed_valuesZprecomputed_elementsZprecomputed_template_parametersZprecomputed_template_params_strZprecompute_template_declZ#precomputed_elements_with_cpp_typesZprecomputed_elements_declZsetter_methodsir0Zreturn_ty_templatesZ return_tyZ elem_cpp_tyrZ assert_msgZ assert_stmtZconstruction_stmtsjZconstruction_blockZsetter_methods_declZmeta_return_template_paramsZmeta_return_typedefZprecomputed_declrVrVrW!compute_meta_function_declarations              rD)rrrecCsBt|jjj}|ds"|dr&dS|jjjdkr8dS||S)NZ_likeZnew_F)rrfrendswith startswithrrr)rrrrVrVrWneeds_backend_selectgs rGc@sJeZdZUeeejeejfed<e ed<e e e e dddZdS)ComputeBackendSelectr rrc Cs4t||jsdSt|j}t|jdd}dd|D}t|j}|}| }d}|j t j kr|r|jj sztddd|D} d |d | d } n|jj rtd |d } d|jd||d| d|jjdddd|Dd S|j t jkr&d|jjd|dSt|j dS)NTrcSs(g|] }t|jtr|jjr|qSrV)rrrr#rrrrVrVrWrs z1ComputeBackendSelect.__call__..z.c10::computeDispatchKey(dtype, layout, device)rcss|] }|jVqdSrrrrVrVrWrsz0ComputeBackendSelect.__call__..z-DispatchKeySet _dk_set = c10::DispatchKeySet(z() | c10::detail::multi_dispatch_key_set(z); DispatchKeySet _dk_mask = c10::DispatchKeySet(DispatchKeySet::FULL_AFTER, DispatchKey::BackendSelect); DispatchKeySet _dk = c10::impl::computeDispatchKeySet(_dk_set, _dk_mask);z)DispatchKeySet _dk = c10::DispatchKeySet(rz // aten::z C10_ALWAYS_INLINE z { z return at::_ops::z::redispatch( _dk, css|] }|jVqdSrrrrVrVrWrsrzm.impl("aten::z ", TORCH_FN(z));)rGrnativerrfrrrrrr rFrZhas_tensor_argrtrrr REGISTRATIONr?) rQrrZ native_sigrZdispatcher_sigrZdispatcher_exprsrnrZ compute_dkrVrVrWr{sJ       zComputeBackendSelect.__call__N)rXrYrZrrrFrrKrr>rr.r rrrVrVrVrWrHss rH)datarecCs:ddt_tttddd}tt|tj|dtddS) NcSsdS)NTrV)rQrLrVrVrWrkrlzformat_yaml..)dumperrLrecSs||Sr)Zrepresent_dictrz)rMrLrVrVrWdict_representersz%format_yaml..dict_representerFgeA)Zdefault_flow_styleZDumperwidth)rGZignore_aliasesrZadd_representerrrdump)rLrNrVrVrW format_yamls  rQc Csh|dkr dS|dkrdSz t|WStk rbzt|WYStk r\|YYSXYnXdS)Nr:TfalseF)rw ValueErrorfloatrrVrVrWpythonify_defaults rU)trecCs:t|trt|jSt|dkr$dStj|ddddS)NTensorz at::TensorF__placeholder__)mutabler5r)rrr2 dynamic_typer0rrZargumenttype_typer9)rVrVrVrWrZs   rZ)r recCs2dg}tj|kr|dtj|kr.|d|S)Nr7rW namespace)r8rrxr)r  method_ofrVrVrWcompute_method_of_yamls     r]cCsi}t|}g}tt|jj|D]f\}\}}t|j|tj|dd d}|j r|j |d<|j r|j ||jj j |j <||q$||fS)NFrI)rZrr field_name)rZ return_namesr?ziprfreturnsrZr return_typer9rr*rr;rx)rname_to_field_namenamesr`rBrrretrVrVrWcompute_returns_yamls*    re)cpp_a schema_orderkwarg_only_set out_arg_setrbrecCsrt|jtr:ddd|j|jdd}|jdk r6|j|d<|St|jtrNtn t|jtrnt |j||||dSdS)Nzat::TensorOptionsFT) annotationrZ is_nullablerr kwarg_onlyrrgrhrirb) rrrr6rrrr4rtr#compute_argument_yaml)rfrgrhrirbrrVrVrWcompute_cpp_argument_yaml>s*     ro)rrgrhrirbrec Cs|jrt|jndt|j|j|jtj|dddd}|j dk r`t t |j |j|d<|j|krrd|d<|j|krd|d<d|d <|j|kr||j|d <|j }|dk r|j dk rt|jd kr|j |d <|S) NrXF)r5r)rjrZrkrrrTrlrallocater^boolsize)rjrrZrrkrrr6r9rrUZ default_exprZ is_list_likerrr0)rrgrhrirbrlrVrVrWrn^s&        rnc st|\}tdd|jjjDtdd|jjjDtj|ddd}|j}fdd|D}t |j }fdd|D}d d|D}t j |jj dd }|d d |d } tdd|Dotj|jk} tdt |jfdt|jjjfdt|jjjfd|jfd|jdk r4|jndfdd|jfd|fd| fd|fdt|jfdd|jdkrxdn|jfd|fd|jjjjfd| fd |jfd!|jfd"d#d$|jfgS)%Ncss|] }|jVqdSrrrrVrVrWrsz+compute_declaration_yaml..css|] }|jVqdSrrrrVrVrWrsFrc sg|]}t|ddqS)Frm)ro)rrfrhrbrirVrWrsz,compute_declaration_yaml..c sg|]}t|ddqS)Trm)rnrrtrVrWrsc Ss0g|](}tj|dtddddD] }|jq qS)F)rZcpp_no_default_argsZfaithfulrZhas_tensor_options)rrrr)rrrrVrVrWrsrIz (rrcss|]}t|jtVqdSr)rrrr6rrVrVrWrsrZ operator_namerrcategory_overriderZ schema_stringaten::rschema_order_cpp_signatureschema_order_argumentsr\)moderJ python_moduler`ris_factory_methodZabstractro)Zwith_gilF) deprecatedFZhas_math_kernel) rerrfrZflat_kwarg_onlyr;rrrrsrxr returns_typer`r9ranyr8rr rrrrrrur]rzrZ is_abstractror) rr`r%Zcpp_argsrZschema_order_jit_argumentsrxZcpp_schema_order_typesZ cpp_returnsrwr{rVrtrWcompute_declaration_yamlsb      rcCs0|js|jdk o.|jtjkp.|jtjkSr)rrrfkindr3 functionalrrrVrVrW"has_autogenerated_composite_kernelsr)rr^rec stj}tjj}tj}ddd|D}djtfdd| Dt j hkofdd| Dt j t j hktj ptd}|d |d |d t|d S) Nrcss|]}|VqdSr)Z no_defaultZdecl_registration_declarationsrrVrVrWrsz4compute_registration_declarations..rvcsh|]\}}|r|qSrVrrrrrrVrW s z4compute_registration_declarations..csh|]\}}|r|qSrVrrrrVrWrs )schemadispatchrr8rz); // r) dispatcherrrfr}r`Z"cpp_type_registration_declarationsrrrrzr(rrZhas_composite_kernelrjsondumps)rr^rr}r@rAZ comment_datarVrrW!compute_registration_declarationss(    r)"provided_op_registration_allowlistop_selection_yaml_pathrecCsd|dk r|dk rtdd}|dk r,t|}|dk rDt|dd}n|dk rXt|}nt}|S)NzhBoth provided_op_registration_allowlist and op_selection_yaml_path can NOT be provided at the same time.TF)rtrr>Z&from_legacy_op_registration_allow_listZfrom_yaml_pathget_nop_selector)rrZop_registration_allowlistrrVrVrWget_custom_build_selectors&  r)r]recCsttttftfttttfddd}tt}|D]b}|j }|j }|tj kr~|j }|||kspt||||<q8|||kst||||<q8tt||S)NdrecSsfg}tj|krJ|tj}|tjd}|tjd}|t|||d|D]}||qR|S)N)view view_copy view_inplace) r9ZaliasingrZaliasing_inplacer3rrxr0r>)rrirrrrfrVrVrWmaybe_create_view_group%s    zEget_grouped_by_view_native_functions..maybe_create_view_group)rrr9r3r.rr0rrurfZview_signatureZview_schema_kindZ non_aliasingrrtrsr@r>)r]rZgrouped_by_viewsrrZ view_kindrrVrVrW$get_grouped_by_view_native_functions"s    rcCsBtttfttttfddd}t|}tt|t| S)NrcSsBt|}|dkr8tdd|Dr,tt|S|gSdS)Ncss|]}d|jkVqdS) generatedN)rrrrVrVrWrZszJget_grouped_native_functions..flatten_pre_group..)r/ from_dictr~r>rtrs)rrrVrVrWflatten_pre_groupSs   z7get_grouped_native_functions..flatten_pre_group) rr3r.r rr/r=rsr@r>)r]rZpre_grouped_native_functionsrVrVrWget_grouped_native_functionsPs  r)grouped_native_functionsr^rec Csg}tt}d}|D]}t}t}|D]l\}} | |} | r^| j} |||| nt} t|dkst d|d|||  t || q,q|D]R\} } t | ddd} tt| }| d| jd||d| jd|q|S) NrrLz6Codegen only supports one namespace per operator, got z from r namespace_str entity_nameZ max_level )rrsrrzrrrr'rrtextenddest#compute_native_function_declarationrErfromkeysprologuerepiloguesplit)rr^ declarationsns_grouped_kernelsnewlinerZnative_function_namespaces dispatch_keysrn backend_idxrr[kernels ns_helperordered_kernelsrVrVrW get_native_function_declarationsgsN      r)rrrecCs^||}|rPd|jksPtdt|tr.|jjn|jjjd|jd|jd|rZ|jSt S)NrzThe kernel for function z with dispatch key z has a namespace z% and it's not ending with '::native'.) rrrtrrr.rfrrrnr')rrrrVrVrWget_kernel_namespaces  2 r) fmrrnrrrocmrskip_dispatcher_op_registrationgen_dispatch_helpersrec sg} tttttt} d} tjtj|||dd} tjtj|||dd} tjtj|||dd}|D]}t |d dd | | | |t |t r|jn|jj}|| krtt| <| | ||qrD]tdkrqtdd| D]B}| |s>q(d|d d | | |d 7q(| |d f d d| q| S)Nrrrclass_method_namerrrrrr)rz TORCH_LIBRARY_IMPL(rz , m) {  };zRegisterDispatchDefinitions.inic s<jjrtngr&dnddS)Nr)Z ns_prologueZ ns_epiloguedispatch_helpersZdispatch_anonymous_definitionsZ"static_init_dispatch_registrationsZdeferred_dispatch_registrationsdispatch_namespaceZdispatch_namespaced_definitions)rrrZgen_registration_helpersrrV Zanonymous_definitionsrrnrZkernel_namespaceZns_definitionsrZregistration_bodyrrVrWrks z1get_native_function_definitions..)rrsrurRegisterDispatchKeyrFZNAMESPACED_DEFINITIONZANONYMOUS_DEFINITIONrKrrrrrr.r[rrrErZsubstitute_with_templater)rrrnrrrrrr definitionsZ registrationsrZns_genZ anonymous_genZreg_genrr[rVrrWget_native_function_definitionss          r)rrnrrrrrec Csg}tt}d}tj|tj||dd|d} |D].} t| |dd|} ||  | | q.| D]`\} } t | dkr|qft | ddd } tt | }| d| jd||d| jd |qf|S) NrF)rrrrrrJrrrrr)rrsrrrFNAMESPACED_DECLARATIONrrrrrzrrErrrrrr)rrnrrrrrrrrfrr[rrrrVrVrWget_namespaced_declaration sR     r)r]schema_selectorrec Cstt}|D]}||j|q d}g}d}|D]L\}}ttt||} |dkr^| }q6|}d} |d|d| | d7}q6||fS)NrZatenrz TORCH_LIBRARY(z , m) { r)rrsr[rxrzrDrr) r]rZns_native_functionsZnative_functionschema_registrationsaten_schema_registrationsZcustom_namespacer[riZschema_registrations_bodytabrVrVrW(get_native_function_schema_registrations?s(  r) r]rstructured_native_functionsstatic_dispatch_idxrr^cpu_fmcuda_fmfunctions_keysrrrec s|d fddddDfddD|d fdd|d  fd d|d  fd dtd |dfdd| D]ntr|n|} |krdd| d dfdd| dd fdd~ qdS)NzNativeMetaFunctions.hcsgtttdS)N)ZNativeMetaFunctions_includesZ NativeMetaFunctions_declarationsrsrDrDrV)rrVrWrkos z(gen_aggregated_headers..cSsg|]}tj|jkr|qSrVr8rr rfnrVrVrWrvs z*gen_aggregated_headers..csg|]}|kr|qSrVrVr)method_native_functionsrVrWrysMethodOperators.hcsgttttjddS)Nr ZMethodOperators_includesZMethodOperators_declarationsrsrDr rFrrV)rrrVrWrk~sz Operators.hcs dgttttjddS)Nz!#include r)ZOperators_includesZOperators_declarationsrrV)non_method_native_functionsrrVrWrks Functions.hcstdgtttdS)Nz#include )rZFunctions_includesZFunctions_declarations)rrsrDrrVr]rrVrWrksrr^zNativeFunctions.hcs dgdS)Nz%#include )ZNativeFunctions_includesZNativeFunctions_declarationsrVrV)rrVrWrksrFunctions_inl.h>DispatchKeyFunctions.hcstdSN)rnZinline_headersrrVrn inl_headersrVrWrksFunctions_inl.hDispatchKeyFunctions_inl.hc s$gtdddS)NT)rrnrrrr)!DispatchKeyFunctions_inl_includesr dispatch_namespaced_declarations)rrrV)r^rnrrrrVrWrks)writerr*write_with_template) r]rrrrr^rrrrrrrV) r^rrnrrrr]rrrrrrWgen_aggregated_headers]sZ          r) r]rrrr^rrops_fmrrrrec  sdtdd|D]} | j| qtdd} |D]} | j | | q6D]\ | dd fdd| dd fd d|  g}d d |D t d kr| d d fddt||d| dd fddqXdD](\ |d fddq| D]|krXqD gD]~\ |  g}t t t j |tj|| dddd|td krql | dddfddqltr|n|}dd|dd fd!d|d"d#fd$d~qD|d%fd&ddS)'NcSsgSrrVrVrVrVrWrkrlz*gen_per_operator_headers..cSsgSrrVrVrVrVrWrkrlz_ops.hz Operator.hcsdttttjdiS)NrrrrV) functionsrrVrWrksz.hz Function.hcs2ttfddddtttdS)Ncs t|dS)Nr)rrrrVrWrksz...r_ops.h>)Zstatic_dispatch_ops_headersZoperator_includesZfunction_definitions)rsrDrrV)rrrrVrWrks  cSs g|]}t|tr|jr|qSrV)rrr/rrrVrVrWrs z,gen_per_operator_headers..rz_meta.hzNativeMetaFunction.hcsdtttiS)NZmeta_function_declarationsrrV)structured_functionsrVrWrk$srz _native.hzNativeFunction.hcsrddngdS)Nrz_meta.h>)Zextra_includesZnative_function_declarationsrVrV)r is_structuredrrVrWrk2s))Z Functionsr)Z OperatorsZ_ops)ZNativeMetaFunctionsZ_meta)ZNativeFunctionsZ_nativecs.dfddtDdgiS)NZ _includescsg|]}d|dqS)r.h>rVrr)suffixrVrWrCs>gen_per_operator_headers....Z _declarations)sortedrrV)categoryfunctions_by_root_namerrVrWrkBs  TFrrz _dispatch.hzDispatchKeyFunction.hcs dS)N)rrrVrV)rrrVrWrkjsrrrrcstdSrrrVrrVrWrkvsrrcsfddtDgdS)Ncsg|]}d|ddqS)rrrrVr)rrVrWrsr)rrr)rrV)dispatch_namesrrVrWrk~s  rcstddDgdS)Ncss0|](\}}tdd|Drd|dVqdS)css|]}tj|jkVqdSrrrrVrVrWrszGgen_per_operator_headers.....rrN)r~)rrrrVrVrWrsz=gen_per_operator_headers....r)rrzrV)rrVrWrks )rrrxrzrrvrrrrrsr@rrrFrr*)r]rrrr^rrrrrrrZgrouped_functions_by_root_namegroupZgrouped_functionsrrV) rrrnrrrrrrrrrrrWgen_per_operator_headerss                    r)r]rarrrrr^core_fmrrrrrrper_operator_headersrec s|r$t|||| | | | | d nt||||| | | | d |dfdd|dfdd|dfd d|d fd dtttfd fd d }|d|tttfd fdd }|d|dS)N) r]rrrr^rrrrrr) r]rrrrr^rrrrrz TensorBody.hcs2ttttjdttttjddS)N)r r )Ztensor_method_declarationsZtensor_method_definitions)rsrDr(rFrrrVrrVrWrks& zgen_headers..zRedispatchFunctions.hcsdtttiS)NZfunction_redispatch_definitions)rsrDr+rVr]rVrWrks  zRegistrationDeclarations.hcsdfddDiS)NZregistration_declarationscsg|]}t|qSrV)rrrrVrWrsz1gen_headers....rVrV)r^r]rVrWrks  zVmapGeneratedPlumbing.hcstSrr!rVrrVrWrkrlrec st}t}D]F}|t|jjj||jjjj|jD]}||jqDq|tddddddddd d d g 8}d d dt|Dd ddt|DdS)NandZand_eqZbitandZbitorZcomplnotZnot_eqorZor_eqxorZxor_eqz \ cSsg|]}d|dqS)z_(aten, rrVrrVrVrWrszBgen_headers..gen_aten_interned_strings..cSsg|]}d|dqS)z_(attr, rrVrrVrVrWrs)Z aten_symbolsZ attr_symbols) rrrrfrrrxrr)attrsrcrfrrrVrWgen_aten_interned_stringss8z.gen_headers..gen_aten_interned_stringszaten_interned_strings.hcsddtiS)NZenum_of_valid_tagsz, )rrrV)rarVrW gen_tags_enumsz"gen_headers..gen_tags_enumz enum_tag.h)rrrrr)r]rarrrrr^rrrrrrrrrrrV)r^r]rrarW gen_headerss^      ' r)r]rr view_groupsrrr^rr cpu_vec_fmrrrrforce_schema_registrationrrrec sdr d| D]̉tr"| n|}rJttd fdd nttdfdd tt} D]*}t|tr|jn|jj}|| |qvt t j k}t | d|d |d d d  fd d D]ډ jjrtsq jjjj t jkr||ksBt|d d d fdd | d d d fdd nRt jkrdrd|d dd fdd ntddq~qttttfd fdd }|d|}|rt}t |d\|d fd!d ttttftd"d#d$}|jd% |fd&d d't id(d)d*d'hd+|d,d-d |d.d/d |d0 fd1d ttttfttttfd2fd3d4 }tt}d5d6t!d7d D}d8d6t!d9d D} D],}|jj|kr2|jj|kr2| |q2|jd:|||d;dd?d@hdA|dBfdCd |dDfdEd dS)FNz#include #include #include #include z#include #include #include #include rcsg}D]}d}|r d}nDt|trJtfdd|DrJd}n|jrdtjtjfkrdd}|sjq| d|j dtjkr| d|j dkr| d|j dd qt t |S) NFTc3s|]}|VqdSrrrrrVrWrTsz=gen_source_files..operator_headers..r _native.h>rrr) rrrr/r~rrr(Metarrxrrr)headersr.Z is_registered)rrnrrrrVrWoperator_headersHs6 z*gen_source_files..operator_headerscs8dg}tjkr|dkr4|dd|S)Nz!#include z#include rr)r(rrx)r )rnrrVrWr rs   T) rrrnrrrrrrZRegisterz.cppzRegisterDispatchKey.cppcs*tr nddtddS)Nr)extra_cuda_headersZexternal_backend_headersZdispatch_headers ops_headersrdispatch_definitions)r*rZgen_registration_headersrV)rrrnr r rrrVrWrksz"gen_source_files..Z UfuncCPU_z UfuncCPU.cppcs"tttdS)N)meta_declarationnative_declarationnative_definitions)rDrrZcompute_ufunc_cpurV)r^rnr.rVrWrksZUfuncCPUKernel_zUfuncCPUKernel.cppcstdS)N)rr)rZcompute_ufunc_cpu_kernelrV)r.rrVrWrksz%#include z$#include Z UfuncCUDA_z.cuz UfuncCUDA.cucs&tttdS)N)r cuda_headersrrr)rDrrZcompute_ufunc_cudarV)r^rrnr.rrVrWrksz unrecognized z for ufunccsLfddD}dd|Dttttj|ttttj|dS)Ncsg|]}t|r|qSrV)rGrrrVrWrs z@gen_source_files..gen_backend_select..cSsg|]}d|jdqS)rrrrrVrVrWrs)r Z!backend_select_method_definitionsZ%backend_select_function_registrations)rsrDrHrFrrK)Z relevant_fns)r]rrVrWgen_backend_selects&   z,gen_source_files..gen_backend_selectzRegisterBackendSelect.cpp)r]rzRegisterSchema.cppcsrgnrgndS)N)rrrVrV)rrrrVrWrks)rrecSs|jSrrrrVrVrWkey_func sz"gen_source_files..key_funcz Operators.cppcs&d|jdgttjd|gdS)Nrrr)r r)rr rFrrrrVrWrk srr r)key_fn env_callableZbase_env num_shards sharded_keysz Functions.cppcSsiSrrVrVrVrVrWrk# rlzTensorMethods.cppcSsiSrrVrVrVrVrWrk% rlzATenOpList.cppcsdtttiS)NZaten_ops)rsrDr,rVrrVrWrk) s r-csBttttfttddd}||t|t|tj dS)Nr-cSs&t|tr\d|jjdd|jjdg}|jdk rX|d|jjdd|jjdg7}|St|trd|jjdd|jjdd|jjdd|jjdg}|jdk r|d|jjdd|jjdg7}|j dk r|d|j jdd|j jdg7}|Sd|jdd|jdgSdS)Nrrr) rrr0rrrr/rr;rrY)r.r rVrVrWgen_op_headers1 s<       zPgen_source_files..functionalization_env_callable..gen_op_headers)r func_definitionsfunc_registrations) rr.r/r0rrrrr(r)r.r)r^rrVrWfunctionalization_env_callable. s  (z8gen_source_files..functionalization_env_callablecSsi|]}|jj|qSrVrfrrrVrVrW o sz$gen_source_files..cSs t|Srrsrr.rVrVrWrkq rlcSsi|]}|jj|qSrVr rrVrVrWr!s scSs t|Srr"r#rVrVrWrkt rlzRegisterFunctionalization.cppr rrZfunc_add_back_views_definitionsZ!func_add_back_views_registrations)rrrrzFunctionalInverses.hcsdttfddiS)NZview_inverse_declarationscs t|Sr)r r#rrVrWrk sz4gen_source_files....)rsrDrV)rrrVrWrk s zCompositeViewCopyKernels.cppcsDddDddDtttttttttdS)NcSs:g|]2}ddd|jdkr$|jgn |j|jgDqS)rcss|]}d|jdVqdS)rrNrrrVrVrWr s@gen_source_files.....N)rrrrr.rVrVrWr s z6gen_source_files....cSs,g|]$}ddd|j|j|jfDqS)rcss.|]&}|dk rd|jkrd|jdVqdS)Nrrr)rrrrVrVrWr s r%)rrrYrr&rVrVrWr s )r Z#CompositeViewCopyKernel_DefinitionsZ(GeneratedCompositeFunctional_DefinitionsZ!GeneratedCompositeOut_Definitions)rsrDrr;r<rV)rrrVrWrk s. )"r*rrrrsrrr.r[rrxrr(rrrr;Zufunc_inner_loopr,rfrCPUrtCUDArrr>rrrr/r0Z write_shardedrr@)r]rrrrrr^rrrrrrrrrrrZns_grouped_native_functionsZgrouped_native_functionr[rrrrrZ all_groupsZstructured_mapZview_maprrV)rrr^rrrnrr rr.rrr]r rrrrrrrrrWgen_source_files#s  *                  9     r))rr]recs|dfdddS)NzDeclarations.yamlcstddDS)NcSsg|] }t|qSrV)rrrVrVrWr sz;gen_declarations_yaml....)rQrVrrVrWrk rlz'gen_declarations_yaml..)r)rr]rVrrWgen_declarations_yaml s r*rcCsttjS)z If you're depending on torchgen out-of-tree, you can use the root to figure out the path to native_functions.yaml )pathlibPath__file__parentresolverVrVrVrWget_torchgen_root sr0cstjdd}|jddddd|jdd d d |jd d dd|jdd dd|jddddd|jdd dd|jdd dd|jdddd|jddd |jd dd!d|jd"dd#d|jd$d d%d|jd&d d'd|jd(tdd)d*d+gd)d*d+gd,d-|tjj}tj j d.}tj j d/}d0d1l m }t}jsv|tjtj|krv||tj=t|||}t|}|j|j}t|} d2d3| D} t|} d4d3| D} jd5} t| jd6d6d7jd8}t|jd6d6d7t| d9}td:}td:}td:}t|d9}tjtj tj!tj"tj#tj$tj%h}jrn|tjj&rfd;d3|D}g}j'rԇfdd+j)kr`t0||d?j1rtj12}|j3}|j4}|d@f|dAf|dBf|dCf|dDffD]0\}}||}|j5||}|6|t|qdS)ENzGenerate ATen source files) descriptionz-sz --source-pathz!path to source directory for ATenz aten/src/ATen)helprz-oz--output-dependenciesz:output a list of dependencies into the given file and exit)r2z --dry-run store_truez5run without writing any files (still updates outputs))actionr2z--per-operator-headersz2generate separate headers per operator in ATen/opsz-dz --install_dirzoutput directoryzbuild/aten/src/ATenz--rocmz=reinterpret CUDA as ROCm/HIP and adjust filepaths accordinglyz--mpsz'Generate MPS registration code when setz--op_registration_whitelist*zfilter op registrations by the whitelist (if set); each item is `namespace`::`operator name` without overload name; e.g.: aten::empty aten::conv2d ...)nargsr2z--op_selection_yaml_pathaDProvide a path to the operator selection (for custom build) YAML that contains the information about the set of selected operators and their categories (training, ...). Each operator is either a full operator name with overload or just a bare operator name. The operator names also contain the namespace prefix (e.g. aten::)z--backend_whitelistzRfilter dispatch backend by the whitelist (if set), e.g.: CPU CUDA QuantizedCPU ...z--static_dispatch_backendz?generate static dispatch code for the specific backend (if set)z!--skip_dispatcher_op_registrationz0Avoid registering operators into the dispatcher.z--force_schema_registrationz}force it to generate schema-only registrations for all ops, includingthose that are not listed on --op_registration_whitelistz --generater sourcesZdeclarations_yamlzGenerate only a subset of files)rr6choicesrr2znative/native_functions.yamlznative/tags.yamlr)rcSsg|]}t|tr|qSrV)rrr/r&rVrVrWrQ s zmain..cSsg|]}t|tr|qSrV)rrr0r&rVrVrWrW s z/coreT)parentsexist_okz/ops)options install_dirr;cs&g|]}t|st|jkr|qSrV)r+rbackend_whitelist)rrr=rVrWr scsg|]}t|qSrV)r(parse)rkeyrrVrWr s)r]rrrrrr^rrrrrrrrrr)r]rarrrrr^rrrrrrrr)r]rrZcpu_vec_Zcore_Zcuda_Zops_)7argparseArgumentParser add_argumentr parse_argsrZop_registration_whitelistrosrcr source_pathtorchgen.modelrrZmpsrr(ZMPSrprrr]r^rrr<r+r,mkdirrCr'r(rrrrr r>Zstatic_dispatch_backendr?generater)rrrrrr*Zoutput_dependenciesr/rstemr.Z write_outputs)parserrZnative_yaml_pathrrrbZ parsed_yamlrar]rrZ!native_functions_with_view_groupsrZcore_install_dirZops_install_dirrrrrrrrr@Zdp_keyZ depfile_pathZ depfile_nameZ depfile_stemrr)varnamercrV)r^r;rWmain sx                         rM__main__)Nr_F)r_)N)rA functoolsrrEr+ collectionsrrrZ dataclassesrtypingrrrr r r r r rrZtyping_extensionsrZtorchgen.api.dispatcherapirZtorchgen.api.metar<Ztorchgen.api.nativerJZtorchgen.api.structuredrZ torchgen.destrZ torchgen.apirZtorchgen.api.translaterZtorchgen.api.typesrrrrrrrZtorchgen.contextrrrrZ#torchgen.gen_functionalization_typerrrr Ztorchgen.gen_vmap_plumbingr"rGr#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9Z#torchgen.native_function_generationr:r;r<r=Z!torchgen.selective_build.selectorr>Ztorchgen.utilsr?r@rArBrCrDrErFrGrHrIrJrrr\objectrrqrr lru_cacherrryrrrrrrrrrrr rr(r+r,rDrGrHrQrUrZr]rerornrrrrrrrrrrrrrrr)r*r,r0rMrXrVrVrVrWs ,    $  d 0 !  .    #     %      L`-| D   D  !  "Y #   /   0   i 4   {  C    / p