U Kc@?@sddlZddlmZmZddlZddlZddlmZGdddejjZ GdddejjZ Gdd d ejjZ Gd d d ejjZ dS) N)OptionalTuple)TensorcseZdZdZejjZdeee ddfdd Z de e e e e fe e e fddd Zdee e e e fd d d ZddZedddZeddZZS)LSTMCellaA quantizable long short-term memory (LSTM) cell. For the description and the argument types, please, refer to :class:`~torch.nn.LSTMCell` Examples:: >>> import torch.nn.quantizable as nnqa >>> rnn = nnqa.LSTMCell(10, 20) >>> input = torch.randn(6, 10) >>> hx = torch.randn(3, 20) >>> cx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): ... hx, cx = rnn(input[i], (hx, cx)) ... output.append(hx) TN input_dim hidden_dimbiasreturncs||d}t||_||_||_tjj|d|fd|i||_tjj|d|fd|i||_ tj jj |_ tj jj |_tj jj |_tj jj |_tj jj |_dS)Ndevicedtyper )super__init__ input_size hidden_sizer torchnnZLinearigateshgatesaoZ quantizedZFloatFunctionalgatesfgate_cx igate_cgatefgate_cx_igate_cgateogate_cyselfrrr r r factory_kwargs __class__G/tmp/pip-unpacked-wheel-gikjz4vx/torch/ao/nn/quantizable/modules/rnn.pyr s    zLSTMCell.__init__)xhiddenr cCs|dks |ddks |ddkr4||jd|j}|\}}||}||}|j||}|dd\}} } } t |}t | } t | } t | } |j | |} |j || } |j| | }|}t |}|j | |}||fS)Nrr)initialize_hiddenshape is_quantizedrrraddchunkrZsigmoidtanhrmulrrr)rr$r%hxcxrrrZ input_gateZ forget_gateZ cell_gateZout_gaterrrcyZtanh_cyZhyr"r"r#forward2s$        zLSTMCell.forwardF) batch_sizer)r cCsVt||jft||jf}}|rNtj|ddtjd}tj|ddtjd}||fS)N?rZscaleZ zero_pointr )rzerosrquantize_per_tensorZquint8)rr2r)hcr"r"r#r'Ks "zLSTMCell.initialize_hiddencCsdS)NZQuantizableLSTMCellr"rr"r"r# _get_nameRszLSTMCell._get_namecCs|dk|dkkst|jd}|jd}||||dk d}tj||j_|dk rbtj||j_tj||j_|dk rtj||j_|S)zUses the weights and biases to create a new LSTM cell. Args: wi, wh: Weights for the input and hidden layers bi, bh: Biases for the input and hidden layers Nr&)rrr ) AssertionErrorr(rr ParameterrZweightr r)clswiwhbibhrrcellr"r"r# from_paramsUs  zLSTMCell.from_paramscCs\t||jkstt|ds$td||j|j|j|j}|j |_ |j |j _ |j |j _ |S)Nqconfigz$The float module must have 'qconfig') type _FLOAT_MODULEr;hasattrrCZ weight_ihZ weight_hhZbias_ihZbias_hhrDrr)r=otherobservedr"r"r# from_floatjs   zLSTMCell.from_float)TNN)N)F)NN)__name__ __module__ __qualname____doc__rrrrFintboolrrrrr1r'r: classmethodrCrJ __classcell__r"r"r r#r s( rcsXeZdZdZd eeeddfdd Zd eee eefddd Z e d d Z Z S)_LSTMSingleLayerzA single one-directional LSTM layer. The difference between a layer and a cell is that the layer can process a sequence, while the cell only expects an instantaneous value. TNrcs0||d}tt||fd|i||_dSNr r )rrrrBrr r"r#r|s  z_LSTMSingleLayer.__init__r$r%cCs<g}|D]}|||}||dqt|d}||fS)Nr)rBappendrstack)rr$r%resultZxxZ result_tensorr"r"r#r1s   z_LSTMSingleLayer.forwardcOs(tj||}||j|j|j}||_|S)N)rrCrrr rB)r=argskwargsrBlayerr"r"r#rCs z_LSTMSingleLayer.from_params)TNN)N)rKrLrMrNrOrPrrrrr1rQrCrRr"r"r r#rSvsrScs^eZdZdZdeeeeeddfdd Zdeee eefdd d Z e dd d Z Z S) _LSTMLayerz#A single bi-directional LSTM layer.TFN)rrr batch_first bidirectionalr c sZ||d}t||_||_t||fd|i||_|jrVt||fd|i||_dSrT)rrr]r^rSlayer_fwlayer_bw) rrrr r]r^r r rr r"r#rs  z_LSTMLayer.__init__rUcCs|jr|dd}|dkr$d\}}n|\}}d}|jr|dkrDd}n|d}|d}|dkrbd}n|d}|d}|dk r|dk r||f}|dkr|dkrd}ntj|tj|f}|||\} }t|dr|jr|d} | | |\} }| d} t | | g| d} |dkr6|dkr6d} d}nh|dkrRtj|\} }nL|dkrntj|\} }n0t |d|dgd} t |d|dgd}n| } tj|\} }|jr| dd| | |ffS)Nrr&)NNr`)r] transposer^rjit_unwrap_optionalr_rGZflipr`catZdimrWZ transpose_)rr$r%Zhx_fwZcx_fwZ hidden_bwZhx_bwZcx_bwZ hidden_fwZ result_fwZ x_reversedZ result_bwrXr7r8r"r"r#r1sP       z_LSTMLayer.forwardrcKs:t|ds|dk st|d|j}|d|j}|d|j}|d|j}|d|j} |||||| } t|d|| _ t|d|} t|d |} t|d |d} t|d |d}t | | | || _ |jr6t|d|d } t|d |d } t|d |d d} t|d |d d}t | | | || _ | S) z There is no FP equivalent of this class. This function is here just to mimic the behavior of the `prepare` within the `torch.ao.quantization` flow. rDNrrr r]r^Z weight_ih_lZ weight_hh_lZ bias_ih_lZ bias_hh_lZ_reverse)rGr;getrrr r]r^getattrrDrSrCr_r`)r=rHZ layer_idxrDrZrrr r]r^r[r>r?r@rAr"r"r#rJs(z_LSTMLayer.from_float)TFFNN)N)rN)rKrLrMrNrOrPrrrrr1rQrJrRr"r"r r#r\s 4r\c s~eZdZdZejjZdeeee e e e ddfdd Z de e ee e fd d d Zd dZedddZeddZZS)LSTMaUA quantizable long short-term memory (LSTM). For the description and the argument types, please, refer to :class:`~torch.nn.LSTM` Attributes: layers : instances of the `_LSTMLayer` .. note:: To access the weights and biases, you need to access them per layer. See examples below. Examples:: >>> import torch.nn.quantizable as nnqa >>> rnn = nnqa.LSTM(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> c0 = torch.randn(2, 3, 20) >>> output, (hn, cn) = rnn(input, (h0, c0)) >>> # To get the weights: >>> # xdoctest: +SKIP >>> print(rnn.layers[0].weight_ih) tensor([[...]]) >>> print(rnn.layers[0].weight_hh) AssertionError: There is no reverse path in the non-bidirectional layer r&TFN)rr num_layersr r]dropoutr^r c s$|| d} t||_||_||_||_||_t||_||_ d|_ |rPdnd} t |t j rd|krtdkrnn t |trtd|dkrtd|dkrtd||t|j|j|jfd|j d | g} td|D],} | t|j|j|jfd|j d | qtj| |_dS) Nr Fr&rzbdropout should be a number in range [0, 1] representing the probability of an element being zeroedz|dropout option for quantizable LSTM is ignored. If you are training, please, use nn.LSTM version followed by `prepare` step.zdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout={} and num_layers={})r]r^)rrrrrir r]floatrjr^Ztraining isinstancenumbersNumberrP ValueErrorwarningswarnformatr\rangerVrrZ ModuleListlayers)rrrrir r]rjr^r r rnum_directionsrur[r r"r#rsP    $   z LSTM.__init__rUcs|jr|dd}|d}|jr&dnd}|dkrtj|||jtj|jd d|j rptj dd|j dfddt |jD}nvtj|}t|dtr|d|j|||jd|d|j|||jdfd dt |jD}n|}g}g}t|jD]D\} } | ||| \}\} } |tj| |tj| qt|} t|}| d | jd | jd } |d |jd |jd }|jr|dd}|| |ffS) Nrr&rk)r r r3r4csg|] }fqSr"r").0_)r5r"r# Isz LSTM.forward..cs(g|] }|d|dfqS)r)squeeze_)rwidx)r/r.r"r#rySs)r]rasizer^rr5rrlr rzr)r6r rtrirbrcrmrZreshapeZunbind enumeraterurVrWr()rr$r%Zmax_batch_sizervZhxcxZhidden_non_optZhx_listZcx_listr{r[r7r8Z hx_tensorZ cx_tensorr")r/r.r5r#r1;s\       z LSTM.forwardcCsdS)NZQuantizableLSTMr"r9r"r"r#r:jszLSTM._get_namecCst||jstt|ds"|s"t||j|j|j|j|j|j |j }t |d||_ t |jD]}tj|||dd|j|<q\|tjjj|dd}|S)NrDF)r]T)Zinplace)rmrFr;rGrrrir r]rjr^rfrDrtr\rJruevalrrZ quantizationprepare)r=rHrDrIr{r"r"r#rJms  zLSTM.from_floatcCs tddS)NzuIt looks like you are trying to convert a non-quantizable LSTM module. Please, see the examples on quantizable LSTMs.)NotImplementedError)r=rHr"r"r# from_observed|szLSTM.from_observed)r&TFrhFNN)N)N)rKrLrMrNrrrgrFrOrPrlrrrrr1r:rQrJrrRr"r"r r#rgs.*/ rg) rntypingrrrqrrrModulerrSr\rgr"r"r"r#s ib