U Kc@sLUddlZddlZddlZddlmZddlmZmZddlm Z m Z ddl m Z ddl mZddlmZddlmZmZmZddlmZdd lmZmZdd lmZmZd d lmZdd lm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(ddl)m*Z*ddl+Z+ddl,Z,dddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+gZ-ejj.j/ejj.j0ejj.j1hZ2ee$e e3d,d-d'Z4ee$e e3d,d.d&Z5d|e6d0d1d!Z7d2dZ8e e!e&e6e'e e6fe#e6e$ffd3d4dZ9d}eejj:ee#e6ejj:fee#e6e&e6e;ffe3e6ed6 d7d*Ze e d;ddZ@e6e d?d@dZAee!e"edAdBdZBee"eedCdDd ZCejj:e$dEdFdZDejj:ee6e$edGdHdZEe6e$e$e#e6ejj:fee#e6e&e6e;ffe&eefdIdJdZFee#e6ejj:fe#ee3fe3dKdLdZGee"eHdAdMdZIe"eHe ege"eHfdNdOd+ZJe*d$dPZKeKdQdRej3eJd geLeJdSgieKdQdTeHeIieKdQdUeHeIieKdQdVeHeIieKdQdWeHeJd gieKdQdXeHeIieKdQejMeHeIieKdQdYeHeJd gieKdQdZeHeJd gieKdQejNeHeJd gieKdQd[eHeIii ZOe#eKe#e'e;ejPfe ege"eHfffeQd(<iZRe#e'e;ejPfe ege"eHffeQd<ee#e'e;ejPfe ege"eHffdAd\dZSee3dAd]d%ZTee3dAd^d"ZUd~ee#e6ej:fe!e(ej:e$e!ed_d`d#ZVee&e$dafeedbdcdZWee3e&e"e6e"e(e$fdddedZXdee#e6ejj:fee!e$e3dfdgdhZYee#e6ejj:fe!ejj:didjdkZZeejj:e#e6ejj:feedldmdnZ[eejj:e#e6ejj:feedldodpZ\ee#e6ejj:fe!edqdrdsZ]edtdudvZ^e'ee fed3dwdxZ_dee e3e3dydzd{Z`dS)N) QConfigAny QuantType) BackendConfigDTypeWithConstraints)FakeQuantizeBase) ObserverBase) DeQuantStub)"activation_is_statically_quantized is_per_tensoris_per_channel)is_activation_post_process) GraphModulemap_arg)GraphNode)PrepareCustomConfig) CallableOptionalListDictAnySetTupleUnionType) namedtupleall_node_args_except_firstall_node_args_have_no_tensorsassert_and_get_unique_devicecollect_producer_nodescreate_getattr_from_value'create_node_from_old_node_preserve_metacreate_qparam_nodesEMPTY_ARG_DICTget_custom_module_class_keysget_linear_prepack_op_for_dtypeget_new_attr_name_with_prefix(get_non_observable_arg_indexes_and_typesget_per_tensor_qparams get_qconv_opget_qconv_prepack_opget_quantize_node_info#get_skipped_module_name_and_classes graph_module_from_producer_nodesgraph_pretty_stris_get_tensor_info_nodemaybe_get_next_moduleNodeInfonode_return_type_is_intnode_arg_is_biasnode_arg_is_weightNON_OBSERVABLE_ARG_DICTNON_QUANTIZABLE_WEIGHT_OPS quantize_nodereturn_arg_list)nodeargbackend_configreturncCspt|trl|jdkrl|j|jkrl|j|jjd}|dk r\|t|jkr\|j||kr\dS|j d|kSdS)zReturns if node arg is weight call_functionweightNTF isinstanceroptargetZconfigsZ_input_type_to_indexgetlenargskwargs)r:r;r<Z weight_indexrHB/tmp/pip-unpacked-wheel-gikjz4vx/torch/ao/quantization/fx/utils.pyr5Gs  $cCspt|trl|jdkrl|j|jkrl|j|jjd}|dk r\|t|jkr\|j||kr\dS|j d|kSdS)zReturns if node arg is biasr>ZbiasNTFr@)r:r;r<Z bias_indexrHrHrIr4Ps  $T)r=cCstd}td}dddddd}i}d }|D]}t|||<q0g}|jD]} t| j} |rl| d d } t| j} |r| |kr|| } t| j} |r| | } | rd | d d} | | }|rd| d d} | d d } t| j }|r|d d }t| j }t || | | ||fD] \}}t||t|||<q|| | | ||gqLd}d}||j||7}|D]}||j||7}qp|r|d7}|S)zyReturns a printable representation of the ops in the graph of g. If shorten is True, tries to abbreviate fields. zz ZplchdrZgt_prmZcl_funZcl_modZcl_meth) placeholderget_attrr> call_module call_method)namerBrCrFrGactivation_post_processZobszz t|jstd|\}}t|}t|}|j}|||fS)Nz)Only per tensor quantization is supported)r qschemeAssertionErrorcalculate_qparamsfloatintdtype)rOscale zero_pointrkrHrHrIr)s  )rOr=c Cs|j}d}t|dr|j}d}|tjtjfkrt|dsd}|\}}t|jrrt |j }||||d}tj }qt |}t |}|||d}tj }nh|tjtjtjfkrd}tj}tjjjdk} || d}n0|tjkrd}d }d |i}ntd |dS|||fS) z Given an activation_post_process module, return node_type(e.g. call_function), quantize op(e.g. quantize_per_tensor) and a dictionary of extracted qparams from the module N compute_dtyper>)_scale_ _zero_point_Z_axis__dtype_)rorprq)Zfbgemmx86)rqZ_reduce_range_rMtorqz?Unsupported activation_post_process in get_quantize_node_info: )rkhasattrrntorchZquint8qint8rhr rfrjch_axisZquantize_per_channelriZquantize_per_tensorfloat16Zquantize_per_tensor_dynamicbackends quantizedZenginewarningswarn) rOrkrn quantize_op node_typerlrmrwqparamsZ reduce_rangerHrHrIr,s<        _output) in_node obs_moduleobs_nodemodulesquantized_graphnode_name_to_scopeis_input output_prefixr=cCs|r\t|j}|r|dnd} d} |D]&} | jdkr&| jtjjjkr&| } qNq&| rV| } d} n|} |} | r| j|kr|| j\} }nd} |d}|}t |}|dk st d||\}}}|g}| D]<\}}|dkrt ||| | ||}| |q| |q|||t|iS)a  Add quantization nodes (eg. quantize_per_tensor/per_channel) for given node to graph with the qparams calculated from activation_post_process (obs_module). The observer node (obs_node) is used to find the FQN of the user of act_post_process. e.g. Given input `node` in `node = self.conv(x)`, insert node: `quantized_node = torch.quantize_per_tensor(x, self._scale_0, self._zer_point_0, self._dtype_0)` where self._scale_0, self._zero_point_0 and self._dtype_0 are calculated from `obs_module` rNr>_inputrQz7Expecting quantize node info not to be None, observer: )rorp)listusersrBrCrunn functionalZlinearrNr,rgitemsr!r[ create_nodetuple)rrrrrrrrrZfirst_linear_use_or_first_useZ linear_noderaprefix module_path_ root_modulegraphZmaybe_quantize_node_infor~r}rinputskeyvalueZ qparam_noderHrHrIr8s<     )custom_module_mappingr=cCsDt}tjtjtjfD]$}||i}t|}||O}qt|S)a Get all the unique custom module keys in the custom config dict e.g. Input: { QuantType.STATIC: { CustomModule1: ObservedCustomModule }, QuantType.DYNAMIC: { CustomModule2: DynamicObservedCustomModule }, QuantType.WEIGHT_ONLY: { CustomModule3: WeightOnlyObservedCustomModule }, } Output: # extract the keys across all inner STATIC, DYNAMIC, and WEIGHT_ONLY dicts [CustomModule1, CustomModule2, CustomModule3] )setrZSTATICZDYNAMICZ WEIGHT_ONLYrDkeysr)rZfloat_custom_module_classesZ quant_modeZquant_mode_custom_module_configZ quant_mode_custom_module_classesrHrHrIr% s    cCs6|tjkrtjjjS|tjkr(tjjjStd|dS)Nz&can't get linear prepack op for dtype:)rurxopsrzZlinear_prepack_fp16rvZlinear_prepack Exception)rkrHrHrIr&&s     )conv_opr=cCsVtjjjtjjjtjjjtjjjtjjj tjjj i}| |d}|sRt d ||S)NzDidn't find prepack op for {})rurrconv1drrzZconv1d_prepackconv2dZconv2d_prepackconv3dZconv3d_prepackrDrgr\)rZ prepack_opsZ prepack_oprHrHrIr+.s )rhas_relur=cCstjjjtjjjtjjjtjjjtjjj tjjj itjjjtjjjtjjjtjjjtjjj tjjj id}|| |}|st d |||S)N)TFz4Can't find corresponding quantized conv op for {} {})rurrrrrzZ conv1d_relurZ conv2d_relurZ conv3d_relurDrgr\)rrZqconv_opZqconvrHrHrIr*8s$ )rr=cs&ddtjjdfdd }|S)N.r)modulecs>tdfdd }d}||}t||r:|d7}||}q|S)Nics t|SN)rUrrrHrI get_attr_nameSszOget_new_attr_name_with_prefix..get_new_attr_name..get_attr_namerr)rjrt)rrr attr_namerrHrIget_new_attr_nameRs  z8get_new_attr_name_with_prefix..get_new_attr_name)rVrurModule)rrrHrrIr'Os  r:r=cCs|g}|g}|r|}t|jt|j}|D]H}t|tsDq4|jdkrTdS|||jdkrr|j t ks4||q4q |S)a Starting from a target node, trace back until we hit inpu or getattr node. This is used to extract the chain of operators starting from getattr to the target node, for example def forward(self, x): observed = self.observer(self.weight) return F.linear(x, observed) collect_producer_nodes(observed) will either return a list of nodes that produces the observed node or None if we can't extract a self contained graph without free variables(inputs of the forward function). rJNr>) poprrFrGvaluesrArrBr[rCgetattr)r:rTZfrontierZall_argsr;rHrHrIr ]s    )rootproducer_nodesr=cslt|dkstd|t}ifdd}|D]}||||<q6|||dt||}|S)a- Construct a graph module from extracted producer nodes from `collect_producer_nodes` function Args: root: the root module for the original graph producer_nodes: a list of nodes we use to construct the graph Return: A graph module constructed from the producer nodes rz'list of producer nodes can not be emptycst|fddS)Ncs|SrrHr:envrHrIzDgraph_module_from_producer_nodes..load_arg..)rarrHrIload_argsz2graph_module_from_producer_nodes..load_arg)rErgreverserZ node_copyoutputr )rrrrZ producer_nodeZ graph_modulerHrrIr.xs   )rr=cCs^dd|Ddd|DB}t|dks>td|t|dkrVtt|nd}|S)z Returns the unique device for a module, or None if no device is found. Throws an error if multiple devices are detected. cSsh|] }|jqSrHdevice.0prHrHrI sz/assert_and_get_unique_device..cSsh|] }|jqSrHrrrHrHrIrsrzMprepare only works with cpu or single-device CUDA modules, but got devices {}rN) parametersbuffersrErgr\nextiter)rZdevicesrrHrHrIrs)rrrrr=c CsZt|}||}t|}t|tjr0|n tj||d}|||| d|}|S)z Given a value of any type, creates a getattr node corresponding to the value and registers the value as a buffer to the module. rrK) r'rrAruZTensorclonedetachZtensorZregister_bufferr) rrrrrrr new_valueZ attr_noderHrHrIr!s   ) node_namerlrmrrrr=c Cs@|d}||\}}t|||d|} t|||d|} | | fS)z Create getattr nodes in the quantized graph for scale and zero point values. The nodes are registered with the root_module of the model. rQrorp)r!) rrlrmrrrrrrZ scale_nodeZzero_point_noderHrHrIr#s  )r:rcacher=c Cs|r||kr||Sd}t|ts*d}n|jdkrrKr)ndimshaperMsize)rArrBrCrUrgr rrFoperatorgetitemrrrj) r:rrreZfound_one_tensorr;Zlist_elZ!this_list_el_args_have_no_tensorsZthis_arg_args_have_no_tensorsrHrHrIrsj               cCsttdt|jS)z2 Returns all node arg indices after first r)rrangerErFrrHrHrIr s) arg_indicesr=cstttdfdd }|S)zu Constructs a function that takes a node as arg and returns the arg_indices that are valid for node.args rcsfddDS)Ncsg|]}|tjkr|qSrH)rErF)rrrrHrI sz=return_arg_list..arg_indices_func..rHrrrrIarg_indices_funcsz)return_arg_list..arg_indices_func)rrrj)rrrHrrIr9sz op targetrMZ masked_fillZpermuterepeatZreshaper transpose unsqueezeZ unsqueeze_viewcCst|j|j}t|tS)z Returns a dict with of non float tensor types as keys and values which correspond to a function to retrieve the list (which takes the node as an argument) )r2rBrCr6rDr$)r:inforHrHrIr(GscCs|jdko|jdkS)zd Returns true if this node results in an integer, even if some of the args are Tensors. rMr)rBrCrrHrHrIr3PscCs&|jdko |jtko |jddk}|S)z Returns True if this node is a node that takes a Tensor as input and output some meta information about the Tensor, e.g. shape, size etc. r>rr)rBrCrrF)r:rerHrHrIr0Xs )r:rtarget_module_typetarget_functional_typer=cCsj|jD]Z\}}|jdkr@|dk r@t|t|j|r@|S|jdkr |dk r |j|kr |Sq dS)a% Gets the next module that matches what is needed in is_target_module_type if it exists Args: node: The node whose users we want to look at target_module_type: Module type that we want to check target_functional_type: Functional type that we want to check rLNr>)rrrBrArUrC)r:rrruserrrHrHrIr1`s .)rcreate_node_argsold_noder=cCs|j|}|j|_|S)zU Creates `new_node` and copies the necessary metadata to it from `old_node`. )rZ stack_trace)rrrZnew_noderHrHrIr"ys )prepare_custom_configis_standalone_moduler=cCsVt|j}t|j}|sN|t|j7}|t|j7}|t|j7}||fSr) copyZnon_traceable_module_namesZnon_traceable_module_classesrZstandalone_module_namesrZstandalone_module_classesr%Zfloat_to_observed_mapping)rrZskipped_module_namesZskipped_module_classesrHrHrIr-s  )r: named_modulesqconfigqhandlerr=cCsft||}|dk rP|dk rPt|tjjjjjs2tt|tj j oNt |oN| St|tjj j j SdS)zD Return whether this refers to the custom module LSTM flow. N) _get_modulerAruZaoZ quantizationZfxZquantization_patternsZQuantizeHandlerrgrZLSTMr Zis_custom_moduleZ quantizable)r:rrrmodrHrHrI_is_custom_module_lstms r)r:rr=cCs.|jdkr&t|j|kr&|t|jSdSdS)zO If `node` refers to a call_module node, return the module, else None. rLN)rBrUrC)r:rrHrHrIrsr)r:modelrrr=c Cs^d}t|}||}t}t||||||<|||||fW5QRSQRXdS)z Attach a `DeQuantStub` to the model and create a node that calls this `DeQuantStub` on the output of `node`, similar to how observers are inserted. Z dequant_stub_N)r'rsetattrinserting_afterrL)r:rrrrZget_new_dequant_stub_nameZdequant_stub_nameZ dequant_stubrHrHrI_insert_dequant_stubs   rc Cs\||&|tj|df}t||||}W5QRX|||tj|df}W5QRX||&|tj|df}t||||}W5QRX||&|tj|df} t| |||} W5QRX|| |t|| gf} W5QRX|| |t|| gf} W5QRXt|jD]&} | |kr(| |kr(| || q(t || S)a8 Insert DeQuantStubs after each internal output node of custom module LSTM. Custom module LSTM outputs are nested tuples of the sturcture (output, (hidden0, hidden1)), Since we cannot dequantize a tuple as a whole, we must first break down the tuple into its components through `getitem`. This function transforms the graph as follows: (1) Split the LSTM node into (output, (hidden0, hidden1)) (2) Insert a DeQuantStub after each internal node (3) Recombine the DeQuantStubs into the same structure as before (4) Reroute all consumers of the original LSTM node and its sub-nodes (e.g. lstm[0]) Before: lstm_output | v original_user(s) After: lstm_output / \ / (getitem) \ / \ v v output hidden | / \ (DeQuantStub) (getitem) | / \ v v v output_dq hidden0 hidden1 | | | | (DeQuantStub) (DeQuantStub) | | | | v v | hidden0_dq hidden1_dq | \ / | (tuple) | \ / | v v | hidden_dq \ / \ (tuple) / v v lstm_output_dq | v original_user(s) For step (4), reroute all users of the original LSTM node(s) as follows: lstm_output -> lstm_output_dq lstm_output[0] -> output_dq lstm_output[1] -> hidden_dq lstm_output[1][0] -> hidden0_dq lstm_output[1][1] -> hidden1_dq Return the node `lstm_output_dq`. rr) rr>rrrrrrrreplace_input_with_reroute_tuple_getitem_pattern)r:rrrrZ output_dqhiddenZhidden0Z hidden0_dqZhidden1Z hidden1_dqZ hidden_dqZlstm_output_dqrrHrHrI3_insert_dequant_stubs_for_custom_module_lstm_outputs(A      r)r;rr=c sfdd}fdd}dd}ddttttd fd d }|||g||||g||||g|||gg}|D]}||}|d k rv|Sqvd S) aC Given an argument of a node, if the argument refers to the path through which the node is a consumer of custom module LSTM, return the custom module LSTM node, or None otherwise. This is used to determine whether a node is a consumer of custom module LSTM, and, if so, skip inserting input observers for this node. This is because custom module LSTM produces quantized outputs, so inserting an input observer for the consumer of custom module LSTM would unnecessarily quantize the outputs again. lstm -> consumer In practice, however, custom module LSTM outputs a tuple (output, (hidden0, hidden1)) with DeQuantStubs attached to each internal node (see `_insert_dequant_stubs_for_custom_module_lstm_output`). This tuple can be consumed in one of four ways: lstm -> getitem -> DeQuantStub -> consumer # consume lstm[0] lstm -> getitem -> getitem -> DeQuantStub -> tuple -> consumer # consume lstm[1] lstm -> getitem -> getitem -> DeQuantStub -> consumer # consume lstm[1][0] or lstm[1][1] lstm -> getitem -> DeQuantStub -> tuple -> consumer # consume lstm Thus, we must match against the above patterns instead of simply checking the parent node to determine whether this node is a consumer of a custom module LSTM. cstt|tSr)rArrrrrHrImatch_dq:sz=_maybe_get_custom_module_lstm_from_node_arg..match_dqcs t|Sr)rrrrHrI match_lstm=sz?_maybe_get_custom_module_lstm_from_node_arg..match_lstmcSs|jdko|jtjkSNr>)rBrCrrrrHrHrI match_getitem@szB_maybe_get_custom_module_lstm_from_node_arg..match_getitemcSs|jdko|jtkSr)rBrCrrrHrHrI match_tupleCsz@_maybe_get_custom_module_lstm_from_node_arg..match_tuple) match_patternr=csZ}t|D]H\}}||s"dS|t|dkr |krJ|jdd}q |jd}q |S)z Traverse up the graph and match the args one by one. If there is a match, return the last matched node, or None otherwise. Nrr) enumeraterErF)rrrmatch)r;rrHrI_match_patternFs zC_maybe_get_custom_module_lstm_from_node_arg.._match_patternN)rrrr) r;rrrrrZall_match_patternsrZ matched_noderH)r;rrrI+_maybe_get_custom_module_lstm_from_node_args      r)rc stttttttttttttdffdfdd g}t}|jD]}|gg||qN|D]~}|d}|d}|jdkr|jt kst |jdkr|jt j kst |j d}|j d|}t|jD]} | ||qqhd S) a Search for patterns where N consecutive `tuple` call_function nodes are followed by N consecutive `getitem` call_function nodes that are "reverses" of the `tuple` nodes. If we find this pattern, reroute the consumers of the last `getitem` to skip these N `tuple` and `getitem` nodes. Before: a b c | \ / \ tuple \ / tuple | getitem(1) | getitem(0) | d After: b | d .)r: index_stackcurrent_patternmatched_patternsseenc st|dkr0t|dkr0|t|||t|f}||krHdS|||jD]}|jdkr|jtkrt |j dD]4\}}||kr~|||||||||q~qX|jdkrX|jt j krXt|dkrX|j d|dkrX| |||||||qX|S)aP Traverse the graph recursively to match for the N-tuple - N-getitem patterns, starting at the given node. We use a stack to keep track of the expected `getitem` indices, since these are reversed from the `tuple` indices. In the above example, the stack after (b -> tuple -> tuple) will be [0, 1], which will be popped by getitem(1) first and then by getitem(0). TODO: traverse upwards from the output and handle the case when tuple is not a separate node, e.g. graph.call_function(operator.getitem, args=(a, (b, c))) rNr>rr)rEr[rclearraddrrBrCrrFrrr) r:rrrrstaterrZuser_arg find_patternsrHrIrs*       z5_reroute_tuple_getitem_pattern..find_patternsrrr>rN)rrrjrrrrTrBrCrrgrrrFrrrr) rrrr:patternZ first_tupleZ last_getitemZlast_getitem_indexZ new_inputrrHrrIrds& -  rcCs&t|tr|St|tst|jSdS)z If `activation_post_process` is an observer, return the observer. If `activation_post_process` is a fake quantize, return the internal observer. N)rArrrgrO)rOrHrHrI*_get_observer_from_activation_post_processs r)rdtype_with_constraints is_activationr=cstttftttdfdd }dks2|jdkr6dS|r@jnj}|rNdnd}d}|dk r|}t |spt |j|jkrdS||||}|S)a Return whether `qconfig` satisfies the following constraints from the backend, specified through the activation and weight DTypeWithConstraints. 1. QConfig specified a quantization range that falls within the backend's, if any 2. QConfig specified a min scale value that is >= the backend's, if any If `is_activation` is True, we check `qconfig.activation`, else we check `qconfig.weight`. If `qconfig` or `dtype_with_constraints.dtype` is None, or the dtypes do not match, return True. )rOr  debug_stringr=c st|}t|dd}t|dd}t|dd}|j}|j}|j} |dk r|dk r|dks^|dkrttd|fdS||ks||krtd|||||fdS| dk r|dkrtd|fdS|| krtd||| fdSd S) NZ quant_minZ quant_maxZepsz@QConfig %s must specify 'quant_min' and 'quant_max', ignoring %sFzQConfig %s quantization range must fall within the backend's: QConfig range = (%s, %s), BackendConfig range = (%s, %s), ignoring %sz*QConfig %s must specify 'eps', ignoring %szdQConfig %s eps (%s) must be greater than or equal to the backend's min scale value (%s), ignoring %sT)rrZquant_min_lower_boundZquant_max_upper_boundZscale_min_lower_boundr{r|) rOr r ZobserverZ app_quant_minZ app_quant_maxZ app_scale_minZbackend_quant_minZbackend_quant_maxZbackend_scale_minrrHrI;_activation_post_process_satisfies_dtype_config_constraintss@    zp_qconfig_satisfies_dtype_config_constraints.._activation_post_process_satisfies_dtype_config_constraintsNT activationr?) rrrrrUboolrkrr?r rg)rr r r Zactivation_post_process_ctrr Zsatisfies_constraintsrOrHr rI+_qconfig_satisfies_dtype_config_constraintss* %   r)T)r)NN)NN)T)arrRruZtorch.nnrZtorch.ao.quantizationrrZ$torch.ao.quantization.backend_configrrZ#torch.ao.quantization.fake_quantizerZtorch.ao.quantization.observerrZtorch.ao.quantization.stubsrZtorch.ao.quantization.utilsr r r Ztorch.ao.quantization.quantizer Ztorch.fxr rZtorch.fx.graphrrZ custom_configrtypingrrrrrrrrr collectionsrrr{__all__rZ layer_normZ group_normZ instance_normr7rr5r4rUr/r)r,rtyper8r%r&r+r*r'r r.rr!r#rrjrr9r2rirrr6rk__annotations__r$r(r3r0r1r"r-rrrrrrrrrHrHrHrIs      ,    B.1 >$   (K"   4%,0         &   _ EZ