U
    <cd                     @   s  d dl Z d dlmZmZ d dlZd dlmZ ddlmZ d dlm	Z	m
Z
mZ d dlmZ ddlmZ d	d
lmZ dddddddddddddddddddddd d!d"d#d$d%d&d'gZG d(d deZG d)d deZG d*d deZG d+d deZG d,d deZG d-d deZG d.d deZG d/d deZG d0d deZG d1d deZG d2d deZG d3d deZG d4d deZ G d5d deZ!G d6d deZ"G d7d deZ#G d8d deZ$G d9d deZ%G d:d deZ&G d;d deZ'G d<d deZ(G d=d  d eZ)G d>d! d!eZ*G d?d" d"eZ+G d@d# d#eZ,G dAd$ d$eZ-G dBd% d%eZ.G dCd& d&eZ/G dDd' d'eZ0dS )E    N)OptionalTuple)Tensor   )NonDynamicallyQuantizableLinear)	constant_xavier_normal_xavier_uniform_)	Parameter)Module   )
functional	ThresholdReLURReLUHardtanhReLU6SigmoidHardsigmoidTanhSiLUMish	HardswishELUCELUSELUGLUGELU
Hardshrink	LeakyReLU
LogSigmoidSoftplus
SoftshrinkMultiheadAttentionPReLUSoftsign
TanhshrinkSoftminSoftmax	Softmax2d
LogSoftmaxc                       sj   e Zd ZU dZdddgZeed< eed< eed< deeedd fdd	Ze	e	d
ddZ
dd Z  ZS )r   a  Thresholds each element of the input Tensor.

    Threshold is defined as:

    .. math::
        y =
        \begin{cases}
        x, &\text{ if } x > \text{threshold} \\
        \text{value}, &\text{ otherwise }
        \end{cases}

    Args:
        threshold: The value to threshold at
        value: The value to replace with
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    Examples::

        >>> m = nn.Threshold(0.1, 20)
        >>> input = torch.randn(2)
        >>> output = m(input)
    	thresholdvalueinplaceFN)r+   r,   r-   returnc                    s$   t t|   || _|| _|| _d S N)superr   __init__r+   r,   r-   )selfr+   r,   r-   	__class__ ?/tmp/pip-unpacked-wheel-gikjz4vx/torch/nn/modules/activation.pyr1   2   s    zThreshold.__init__inputr.   c                 C   s   t || j| j| jS r/   )Fr+   r,   r-   r2   r8   r5   r5   r6   forward9   s    zThreshold.forwardc                 C   s    | j r
dnd}d| j| j|S )N, inplace=True zthreshold={}, value={}{})r-   formatr+   r,   r2   Zinplace_strr5   r5   r6   
extra_repr<   s      zThreshold.extra_repr)F__name__
__module____qualname____doc____constants__float__annotations__boolr1   r   r;   r@   __classcell__r5   r5   r3   r6   r      s   

c                       sV   e Zd ZU dZdgZeed< ded fddZeeddd	Z	e
d
ddZ  ZS )r   a  Applies the rectified linear unit function element-wise:

    :math:`\text{ReLU}(x) = (x)^+ = \max(0, x)`

    Args:
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/ReLU.png

    Examples::

        >>> m = nn.ReLU()
        >>> input = torch.randn(2)
        >>> output = m(input)


      An implementation of CReLU - https://arxiv.org/abs/1603.05201

        >>> m = nn.ReLU()
        >>> input = torch.randn(2).unsqueeze(0)
        >>> output = torch.cat((m(input),m(-input)))
    r-   Fr-   c                    s   t t|   || _d S r/   )r0   r   r1   r-   r2   r-   r3   r5   r6   r1   a   s    zReLU.__init__r7   c                 C   s   t j|| jdS NrK   )r9   Zrelur-   r:   r5   r5   r6   r;   e   s    zReLU.forwardr.   c                 C   s   | j r
dnd}|S Nzinplace=Truer=   rK   r?   r5   r5   r6   r@   h   s    zReLU.extra_repr)FrB   rC   rD   rE   rF   rI   rH   r1   r   r;   strr@   rJ   r5   r5   r3   r6   r   C   s   
c                       sh   e Zd ZU dZdddgZeed< eed< eed< deeed fd	d
Ze	e	dddZ
dd Z  ZS )r   a  Applies the randomized leaky rectified liner unit function, element-wise,
    as described in the paper:

    `Empirical Evaluation of Rectified Activations in Convolutional Network`_.

    The function is defined as:

    .. math::
        \text{RReLU}(x) =
        \begin{cases}
            x & \text{if } x \geq 0 \\
            ax & \text{ otherwise }
        \end{cases}

    where :math:`a` is randomly sampled from uniform distribution
    :math:`\mathcal{U}(\text{lower}, \text{upper})`.

     See: https://arxiv.org/pdf/1505.00853.pdf

    Args:
        lower: lower bound of the uniform distribution. Default: :math:`\frac{1}{8}`
        upper: upper bound of the uniform distribution. Default: :math:`\frac{1}{3}`
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/RReLU.png

    Examples::

        >>> m = nn.RReLU(0.1, 0.3)
        >>> input = torch.randn(2)
        >>> output = m(input)

    .. _`Empirical Evaluation of Rectified Activations in Convolutional Network`:
        https://arxiv.org/abs/1505.00853
    lowerupperr-         ?UUUUUU?F)rR   rS   r-   c                    s$   t t|   || _|| _|| _d S r/   )r0   r   r1   rR   rS   r-   )r2   rR   rS   r-   r3   r5   r6   r1      s    zRReLU.__init__r7   c                 C   s   t || j| j| j| jS r/   )r9   ZrrelurR   rS   trainingr-   r:   r5   r5   r6   r;      s    zRReLU.forwardc                 C   s    | j r
dnd}d| j| j|S )Nr<   r=   zlower={}, upper={}{})r-   r>   rR   rS   r?   r5   r5   r6   r@      s    zRReLU.extra_repr)rT   rU   FrA   r5   r5   r3   r6   r   m   s   
'
   c                       s|   e Zd ZU dZdddgZeed< eed< eed< deeeee ee dd	 fd
dZ	e
e
dddZedddZ  ZS )r   a  Applies the HardTanh function element-wise.

    HardTanh is defined as:

    .. math::
        \text{HardTanh}(x) = \begin{cases}
            \text{max\_val} & \text{ if } x > \text{ max\_val } \\
            \text{min\_val} & \text{ if } x < \text{ min\_val } \\
            x & \text{ otherwise } \\
        \end{cases}

    Args:
        min_val: minimum value of the linear region range. Default: -1
        max_val: maximum value of the linear region range. Default: 1
        inplace: can optionally do the operation in-place. Default: ``False``

    Keyword arguments :attr:`min_value` and :attr:`max_value`
    have been deprecated in favor of :attr:`min_val` and :attr:`max_val`.

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Hardtanh.png

    Examples::

        >>> m = nn.Hardtanh(-2, 2)
        >>> input = torch.randn(2)
        >>> output = m(input)
    min_valmax_valr-               ?FN)rW   rX   r-   	min_value	max_valuer.   c                    s`   t t|   |d k	r$td |}|d k	r:td |}|| _|| _|| _| j| jks\td S )Nz>keyword argument min_value is deprecated and rename to min_valz>keyword argument max_value is deprecated and rename to max_val)	r0   r   r1   warningswarnrW   rX   r-   AssertionError)r2   rW   rX   r-   r[   r\   r3   r5   r6   r1      s    

zHardtanh.__init__r7   c                 C   s   t || j| j| jS r/   )r9   ZhardtanhrW   rX   r-   r:   r5   r5   r6   r;      s    zHardtanh.forwardrN   c                 C   s    | j r
dnd}d| j| j|S )Nr<   r=   zmin_val={}, max_val={}{})r-   r>   rW   rX   r?   r5   r5   r6   r@      s      zHardtanh.extra_repr)rY   rZ   FNN)rB   rC   rD   rE   rF   rG   rH   rI   r   r1   r   r;   rQ   r@   rJ   r5   r5   r3   r6   r      s(   

     c                       s6   e Zd ZdZd	ed fddZedddZ  ZS )
r   a  Applies the element-wise function:

    .. math::
        \text{ReLU6}(x) = \min(\max(0,x), 6)

    Args:
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/ReLU6.png

    Examples::

        >>> m = nn.ReLU6()
        >>> input = torch.randn(2)
        >>> output = m(input)
    FrK   c                    s   t t| dd| d S )N        g      @)r0   r   r1   rL   r3   r5   r6   r1   	  s    zReLU6.__init__rN   c                 C   s   | j r
dnd}|S rO   rK   r?   r5   r5   r6   r@     s    zReLU6.extra_repr)F)	rB   rC   rD   rE   rI   r1   rQ   r@   rJ   r5   r5   r3   r6   r      s   c                   @   s    e Zd ZdZeedddZdS )r   a  Applies the element-wise function:

    .. math::
        \text{Sigmoid}(x) = \sigma(x) = \frac{1}{1 + \exp(-x)}


    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Sigmoid.png

    Examples::

        >>> m = nn.Sigmoid()
        >>> input = torch.randn(2)
        >>> output = m(input)
    r7   c                 C   s
   t |S r/   )torchZsigmoidr:   r5   r5   r6   r;   %  s    zSigmoid.forwardNrB   rC   rD   rE   r   r;   r5   r5   r5   r6   r     s   c                       sJ   e Zd ZU dZdgZeed< dedd fddZeedd	d
Z	  Z
S )r   a  Applies the Hardsigmoid function element-wise.

    Hardsigmoid is defined as:

    .. math::
        \text{Hardsigmoid}(x) = \begin{cases}
            0 & \text{if~} x \le -3, \\
            1 & \text{if~} x \ge +3, \\
            x / 6 + 1 / 2 & \text{otherwise}
        \end{cases}

    Args:
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Hardsigmoid.png

    Examples::

        >>> m = nn.Hardsigmoid()
        >>> input = torch.randn(2)
        >>> output = m(input)
    r-   FNr-   r.   c                    s   t t|   || _d S r/   )r0   r   r1   r-   rL   r3   r5   r6   r1   H  s    zHardsigmoid.__init__r7   c                 C   s   t || jS r/   )r9   Zhardsigmoidr-   r:   r5   r5   r6   r;   L  s    zHardsigmoid.forward)FrB   rC   rD   rE   rF   rI   rH   r1   r   r;   rJ   r5   r5   r3   r6   r   )  s
   
c                   @   s    e Zd ZdZeedddZdS )r   a  Applies the Hyperbolic Tangent (Tanh) function element-wise.

    Tanh is defined as:

    .. math::
        \text{Tanh}(x) = \tanh(x) = \frac{\exp(x) - \exp(-x)} {\exp(x) + \exp(-x)}

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Tanh.png

    Examples::

        >>> m = nn.Tanh()
        >>> input = torch.randn(2)
        >>> output = m(input)
    r7   c                 C   s
   t |S r/   )ra   tanhr:   r5   r5   r6   r;   e  s    zTanh.forwardNrb   r5   r5   r5   r6   r   P  s   c                       sV   e Zd ZU dZdgZeed< ded fddZeeddd	Z	e
d
ddZ  ZS )r   a  Applies the Sigmoid Linear Unit (SiLU) function, element-wise.
    The SiLU function is also known as the swish function.

    .. math::
        \text{silu}(x) = x * \sigma(x), \text{where } \sigma(x) \text{ is the logistic sigmoid.}

    .. note::
        See `Gaussian Error Linear Units (GELUs) <https://arxiv.org/abs/1606.08415>`_
        where the SiLU (Sigmoid Linear Unit) was originally coined, and see
        `Sigmoid-Weighted Linear Units for Neural Network Function Approximation
        in Reinforcement Learning <https://arxiv.org/abs/1702.03118>`_ and `Swish:
        a Self-Gated Activation Function <https://arxiv.org/abs/1710.05941v1>`_
        where the SiLU was experimented with later.

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/SiLU.png

    Examples::

        >>> m = nn.SiLU()
        >>> input = torch.randn(2)
        >>> output = m(input)
    r-   FrK   c                    s   t t|   || _d S r/   )r0   r   r1   r-   rL   r3   r5   r6   r1     s    zSiLU.__init__r7   c                 C   s   t j|| jdS rM   )r9   Zsilur-   r:   r5   r5   r6   r;     s    zSiLU.forwardrN   c                 C   s   | j r
dnd}|S rO   rK   r?   r5   r5   r6   r@     s    zSiLU.extra_repr)FrP   r5   r5   r3   r6   r   h  s   
c                       sV   e Zd ZU dZdgZeed< ded fddZeeddd	Z	e
d
ddZ  ZS )r   aw  Applies the Mish function, element-wise.
    Mish: A Self Regularized Non-Monotonic Neural Activation Function.

    .. math::
        \text{Mish}(x) = x * \text{Tanh}(\text{Softplus}(x))

    .. note::
        See `Mish: A Self Regularized Non-Monotonic Neural Activation Function <https://arxiv.org/abs/1908.08681>`_

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Mish.png

    Examples::

        >>> m = nn.Mish()
        >>> input = torch.randn(2)
        >>> output = m(input)
    r-   FrK   c                    s   t t|   || _d S r/   )r0   r   r1   r-   rL   r3   r5   r6   r1     s    zMish.__init__r7   c                 C   s   t j|| jdS rM   )r9   Zmishr-   r:   r5   r5   r6   r;     s    zMish.forwardrN   c                 C   s   | j r
dnd}|S rO   rK   r?   r5   r5   r6   r@     s    zMish.extra_repr)FrP   r5   r5   r3   r6   r     s   
c                       sJ   e Zd ZU dZdgZeed< dedd fddZeedd	d
Z	  Z
S )r   a  Applies the Hardswish function, element-wise, as described in the paper:
    `Searching for MobileNetV3 <https://arxiv.org/abs/1905.02244>`_.

    Hardswish is defined as:

    .. math::
        \text{Hardswish}(x) = \begin{cases}
            0 & \text{if~} x \le -3, \\
            x & \text{if~} x \ge +3, \\
            x \cdot (x + 3) /6 & \text{otherwise}
        \end{cases}

    Args:
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Hardswish.png

    Examples::

        >>> m = nn.Hardswish()
        >>> input = torch.randn(2)
        >>> output = m(input)
    r-   FNrc   c                    s   t t|   || _d S r/   )r0   r   r1   r-   rL   r3   r5   r6   r1     s    zHardswish.__init__r7   c                 C   s   t || jS r/   )r9   Z	hardswishr-   r:   r5   r5   r6   r;     s    zHardswish.forward)Frd   r5   r5   r3   r6   r     s
   
c                       sd   e Zd ZU dZddgZeed< eed< deedd fdd	Ze	e	d
ddZ
edddZ  ZS )r   an  Applies the Exponential Linear Unit (ELU) function, element-wise, as described
    in the paper: `Fast and Accurate Deep Network Learning by Exponential Linear
    Units (ELUs) <https://arxiv.org/abs/1511.07289>`__.

    ELU is defined as:

    .. math::
        \text{ELU}(x) = \begin{cases}
        x, & \text{ if } x > 0\\
        \alpha * (\exp(x) - 1), & \text{ if } x \leq 0
        \end{cases}

    Args:
        alpha: the :math:`\alpha` value for the ELU formulation. Default: 1.0
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/ELU.png

    Examples::

        >>> m = nn.ELU()
        >>> input = torch.randn(2)
        >>> output = m(input)
    alphar-   rZ   FNrf   r-   r.   c                    s   t t|   || _|| _d S r/   )r0   r   r1   rf   r-   r2   rf   r-   r3   r5   r6   r1     s    zELU.__init__r7   c                 C   s   t || j| jS r/   )r9   Zelurf   r-   r:   r5   r5   r6   r;     s    zELU.forwardrN   c                 C   s   | j r
dnd}d| j|S Nr<   r=   z
alpha={}{}r-   r>   rf   r?   r5   r5   r6   r@     s    zELU.extra_repr)rZ   FrB   rC   rD   rE   rF   rG   rH   rI   r1   r   r;   rQ   r@   rJ   r5   r5   r3   r6   r     s   
c                       sd   e Zd ZU dZddgZeed< eed< deedd fdd	Ze	e	d
ddZ
edddZ  ZS )r   a.  Applies the element-wise function:

    .. math::
        \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))

    More details can be found in the paper `Continuously Differentiable Exponential Linear Units`_ .

    Args:
        alpha: the :math:`\alpha` value for the CELU formulation. Default: 1.0
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/CELU.png

    Examples::

        >>> m = nn.CELU()
        >>> input = torch.randn(2)
        >>> output = m(input)

    .. _`Continuously Differentiable Exponential Linear Units`:
        https://arxiv.org/abs/1704.07483
    rf   r-   rZ   FNrg   c                    s   t t|   || _|| _d S r/   )r0   r   r1   rf   r-   rh   r3   r5   r6   r1   *  s    zCELU.__init__r7   c                 C   s   t || j| jS r/   )r9   Zcelurf   r-   r:   r5   r5   r6   r;   /  s    zCELU.forwardrN   c                 C   s   | j r
dnd}d| j|S ri   rj   r?   r5   r5   r6   r@   2  s    zCELU.extra_repr)rZ   Frk   r5   r5   r3   r6   r     s   
c                       sX   e Zd ZU dZdgZeed< dedd fddZeedd	d
Z	e
dddZ  ZS )r   ay  Applied element-wise, as:

    .. math::
        \text{SELU}(x) = \text{scale} * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))

    with :math:`\alpha = 1.6732632423543772848170429916717` and
    :math:`\text{scale} = 1.0507009873554804934193349852946`.

    .. warning::
        When using ``kaiming_normal`` or ``kaiming_normal_`` for initialisation,
        ``nonlinearity='linear'`` should be used instead of ``nonlinearity='selu'``
        in order to get `Self-Normalizing Neural Networks`_.
        See :func:`torch.nn.init.calculate_gain` for more information.

    More details can be found in the paper `Self-Normalizing Neural Networks`_ .

    Args:
        inplace (bool, optional): can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/SELU.png

    Examples::

        >>> m = nn.SELU()
        >>> input = torch.randn(2)
        >>> output = m(input)

    .. _Self-Normalizing Neural Networks: https://arxiv.org/abs/1706.02515
    r-   FNrc   c                    s   t t|   || _d S r/   )r0   r   r1   r-   rL   r3   r5   r6   r1   \  s    zSELU.__init__r7   c                 C   s   t || jS r/   )r9   Zselur-   r:   r5   r5   r6   r;   `  s    zSELU.forwardrN   c                 C   s   | j r
dnd}|S rO   rK   r?   r5   r5   r6   r@   c  s    zSELU.extra_repr)FrP   r5   r5   r3   r6   r   7  s   
!c                       sX   e Zd ZU dZdgZeed< dedd fddZeedd	d
Z	e
dddZ  ZS )r   a3  Applies the gated linear unit function
    :math:`{GLU}(a, b)= a \otimes \sigma(b)` where :math:`a` is the first half
    of the input matrices and :math:`b` is the second half.

    Args:
        dim (int): the dimension on which to split the input. Default: -1

    Shape:
        - Input: :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(\ast_1, M, \ast_2)` where :math:`M=N/2`

    Examples::

        >>> m = nn.GLU()
        >>> input = torch.randn(4, 2)
        >>> output = m(input)
    dimNrl   r.   c                    s   t t|   || _d S r/   )r0   r   r1   rl   r2   rl   r3   r5   r6   r1   ~  s    zGLU.__init__r7   c                 C   s   t || jS r/   )r9   Zglurl   r:   r5   r5   r6   r;     s    zGLU.forwardrN   c                 C   s   d | jS )Nzdim={}r>   rl   r2   r5   r5   r6   r@     s    zGLU.extra_repr)rm   rB   rC   rD   rE   rF   intrH   r1   r   r;   rQ   r@   rJ   r5   r5   r3   r6   r   h  s   
c                       sX   e Zd ZU dZdgZeed< dedd fddZeedd	d
Z	edddZ
  ZS )r   a/  Applies the Gaussian Error Linear Units function:

    .. math:: \text{GELU}(x) = x * \Phi(x)

    where :math:`\Phi(x)` is the Cumulative Distribution Function for Gaussian Distribution.

    When the approximate argument is 'tanh', Gelu is estimated with:

    .. math:: \text{GELU}(x) = 0.5 * x * (1 + \text{Tanh}(\sqrt(2 / \pi) * (x + 0.044715 * x^3)))

    Args:
        approximate (str, optional): the gelu approximation algorithm to use:
            ``'none'`` | ``'tanh'``. Default: ``'none'``

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/GELU.png

    Examples::

        >>> m = nn.GELU()
        >>> input = torch.randn(2)
        >>> output = m(input)
    approximatenoneN)rt   r.   c                    s   t t|   || _d S r/   )r0   r   r1   rt   )r2   rt   r3   r5   r6   r1     s    zGELU.__init__r7   c                 C   s   t j|| jdS )N)rt   )r9   Zgelurt   r:   r5   r5   r6   r;     s    zGELU.forwardrN   c                 C   s   d t| jS )Nzapproximate={})r>   reprrt   rq   r5   r5   r6   r@     s    zGELU.extra_repr)ru   )rB   rC   rD   rE   rF   rQ   rH   r1   r   r;   r@   rJ   r5   r5   r3   r6   r     s   
c                       sX   e Zd ZU dZdgZeed< dedd fddZeedd	d
Z	e
dddZ  ZS )r   a  Applies the Hard Shrinkage (Hardshrink) function element-wise.

    Hardshrink is defined as:

    .. math::
        \text{HardShrink}(x) =
        \begin{cases}
        x, & \text{ if } x > \lambda \\
        x, & \text{ if } x < -\lambda \\
        0, & \text{ otherwise }
        \end{cases}

    Args:
        lambd: the :math:`\lambda` value for the Hardshrink formulation. Default: 0.5

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Hardshrink.png

    Examples::

        >>> m = nn.Hardshrink()
        >>> input = torch.randn(2)
        >>> output = m(input)
    lambd      ?Nrw   r.   c                    s   t t|   || _d S r/   )r0   r   r1   rw   r2   rw   r3   r5   r6   r1     s    zHardshrink.__init__r7   c                 C   s   t || jS r/   )r9   Z
hardshrinkrw   r:   r5   r5   r6   r;     s    zHardshrink.forwardrN   c                 C   s   d | jS )Nz{})r>   rw   rq   r5   r5   r6   r@     s    zHardshrink.extra_repr)rx   rB   rC   rD   rE   rF   rG   rH   r1   r   r;   rQ   r@   rJ   r5   r5   r3   r6   r     s   
c                       sd   e Zd ZU dZddgZeed< eed< deedd fdd	Ze	e	d
ddZ
edddZ  ZS )r   a?  Applies the element-wise function:

    .. math::
        \text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)


    or

    .. math::
        \text{LeakyReLU}(x) =
        \begin{cases}
        x, & \text{ if } x \geq 0 \\
        \text{negative\_slope} \times x, & \text{ otherwise }
        \end{cases}

    Args:
        negative_slope: Controls the angle of the negative slope. Default: 1e-2
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(*)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(*)`, same shape as the input

    .. image:: ../scripts/activation_images/LeakyReLU.png

    Examples::

        >>> m = nn.LeakyReLU(0.1)
        >>> input = torch.randn(2)
        >>> output = m(input)
    r-   negative_slope{Gz?FN)r|   r-   r.   c                    s   t t|   || _|| _d S r/   )r0   r   r1   r|   r-   )r2   r|   r-   r3   r5   r6   r1     s    zLeakyReLU.__init__r7   c                 C   s   t || j| jS r/   )r9   Z
leaky_relur|   r-   r:   r5   r5   r6   r;     s    zLeakyReLU.forwardrN   c                 C   s   | j r
dnd}d| j|S )Nr<   r=   znegative_slope={}{})r-   r>   r|   r?   r5   r5   r6   r@   	  s    zLeakyReLU.extra_repr)r}   F)rB   rC   rD   rE   rF   rI   rH   rG   r1   r   r;   rQ   r@   rJ   r5   r5   r3   r6   r     s   
 c                   @   s    e Zd ZdZeedddZdS )r    a  Applies the element-wise function:

    .. math::
        \text{LogSigmoid}(x) = \log\left(\frac{ 1 }{ 1 + \exp(-x)}\right)

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/LogSigmoid.png

    Examples::

        >>> m = nn.LogSigmoid()
        >>> input = torch.randn(2)
        >>> output = m(input)
    r7   c                 C   s
   t |S r/   )r9   Z
logsigmoidr:   r5   r5   r6   r;   !  s    zLogSigmoid.forwardNrb   r5   r5   r5   r6   r      s   c                       sd   e Zd ZU dZddgZeed< eed< deedd fdd	Zeed
ddZ	e
dddZ  ZS )r!   an  Applies the Softplus function :math:`\text{Softplus}(x) = \frac{1}{\beta} *
    \log(1 + \exp(\beta * x))` element-wise.

    SoftPlus is a smooth approximation to the ReLU function and can be used
    to constrain the output of a machine to always be positive.

    For numerical stability the implementation reverts to the linear function
    when :math:`input \times \beta > threshold`.

    Args:
        beta: the :math:`\beta` value for the Softplus formulation. Default: 1
        threshold: values above this revert to a linear function. Default: 20

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Softplus.png

    Examples::

        >>> m = nn.Softplus()
        >>> input = torch.randn(2)
        >>> output = m(input)
    betar+   r      N)r~   r+   r.   c                    s   t t|   || _|| _d S r/   )r0   r!   r1   r~   r+   )r2   r~   r+   r3   r5   r6   r1   C  s    zSoftplus.__init__r7   c                 C   s   t || j| jS r/   )r9   Zsoftplusr~   r+   r:   r5   r5   r6   r;   H  s    zSoftplus.forwardrN   c                 C   s   d | j| jS )Nzbeta={}, threshold={})r>   r~   r+   rq   r5   r5   r6   r@   K  s    zSoftplus.extra_repr)r   r   rr   r5   r5   r3   r6   r!   %  s   
c                       sX   e Zd ZU dZdgZeed< dedd fddZeedd	d
Z	e
dddZ  ZS )r"   a  Applies the soft shrinkage function elementwise:

    .. math::
        \text{SoftShrinkage}(x) =
        \begin{cases}
        x - \lambda, & \text{ if } x > \lambda \\
        x + \lambda, & \text{ if } x < -\lambda \\
        0, & \text{ otherwise }
        \end{cases}

    Args:
        lambd: the :math:`\lambda` (must be no less than zero) value for the Softshrink formulation. Default: 0.5

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Softshrink.png

    Examples::

        >>> m = nn.Softshrink()
        >>> input = torch.randn(2)
        >>> output = m(input)
    rw   rx   Nry   c                    s   t t|   || _d S r/   )r0   r"   r1   rw   rz   r3   r5   r6   r1   l  s    zSoftshrink.__init__r7   c                 C   s   t || jS r/   )r9   Z
softshrinkrw   r:   r5   r5   r6   r;   p  s    zSoftshrink.forwardrN   c                 C   s
   t | jS r/   )rQ   rw   rq   r5   r5   r6   r@   s  s    zSoftshrink.extra_repr)rx   r{   r5   r5   r3   r6   r"   O  s   
c                       s   e Zd ZU dZdgZeej ed< eej ed< ddd	 fd
dZ	dd Z
 fddZdeeeee eee eeeee f dddZ  ZS )r#   aw  Allows the model to jointly attend to information
    from different representation subspaces as described in the paper:
    `Attention Is All You Need <https://arxiv.org/abs/1706.03762>`_.

    Multi-Head Attention is defined as:

    .. math::
        \text{MultiHead}(Q, K, V) = \text{Concat}(head_1,\dots,head_h)W^O

    where :math:`head_i = \text{Attention}(QW_i^Q, KW_i^K, VW_i^V)`.

    ``forward()`` will use a special optimized implementation if all of the following
    conditions are met:

    - self attention is being computed (i.e., ``query``, ``key``, and ``value`` are the same tensor. This
      restriction will be loosened in the future.)
    - Either autograd is disabled (using ``torch.inference_mode`` or ``torch.no_grad``) or no tensor argument ``requires_grad``
    - training is disabled (using ``.eval()``)
    - dropout is 0
    - ``add_bias_kv`` is ``False``
    - ``add_zero_attn`` is ``False``
    - ``batch_first`` is ``True`` and the input is batched
    - ``kdim`` and ``vdim`` are equal to ``embed_dim``
    - at most one of ``key_padding_mask`` or ``attn_mask`` is passed
    - if a `NestedTensor <https://pytorch.org/docs/stable/nested.html>`_ is passed, neither ``key_padding_mask``
      nor ``attn_mask`` is passed

    If the optimized implementation is in use, a
    `NestedTensor <https://pytorch.org/docs/stable/nested.html>`_ can be passed for
    ``query``/``key``/``value`` to represent padding more efficiently than using a
    padding mask. In this case, a `NestedTensor <https://pytorch.org/docs/stable/nested.html>`_
    will be returned, and an additional speedup proportional to the fraction of the input
    that is padding can be expected.

    Args:
        embed_dim: Total dimension of the model.
        num_heads: Number of parallel attention heads. Note that ``embed_dim`` will be split
            across ``num_heads`` (i.e. each head will have dimension ``embed_dim // num_heads``).
        dropout: Dropout probability on ``attn_output_weights``. Default: ``0.0`` (no dropout).
        bias: If specified, adds bias to input / output projection layers. Default: ``True``.
        add_bias_kv: If specified, adds bias to the key and value sequences at dim=0. Default: ``False``.
        add_zero_attn: If specified, adds a new batch of zeros to the key and value sequences at dim=1.
            Default: ``False``.
        kdim: Total number of features for keys. Default: ``None`` (uses ``kdim=embed_dim``).
        vdim: Total number of features for values. Default: ``None`` (uses ``vdim=embed_dim``).
        batch_first: If ``True``, then the input and output tensors are provided
            as (batch, seq, feature). Default: ``False`` (seq, batch, feature).

    Examples::

        >>> # xdoctest: +SKIP
        >>> multihead_attn = nn.MultiheadAttention(embed_dim, num_heads)
        >>> attn_output, attn_output_weights = multihead_attn(query, key, value)

    batch_firstbias_kbias_vr`   TFNrN   c                    s  |
|d}t t|   || _|d k	r*|n|| _|d k	r<|n|| _| j|koT| j|k| _|| _|| _|	| _	|| | _
| j
| | jkstd| jsttj||ff|| _ttj|| jff|| _ttj|| jff|| _| dd  n@ttjd| |ff|| _| dd  | dd  | dd  |rLttjd| f|| _n| dd  t||fd	|i|| _|rttjd
d
|ff|| _ttjd
d
|ff|| _nd  | _| _|| _|   d S )Ndevicedtypez(embed_dim must be divisible by num_headsin_proj_weight   q_proj_weightk_proj_weightv_proj_weightin_proj_biasbiasr   )r0   r#   r1   	embed_dimkdimvdim_qkv_same_embed_dim	num_headsdropoutr   Zhead_dimr_   r
   ra   emptyr   r   r   Zregister_parameterr   r   r   out_projr   r   add_zero_attn_reset_parameters)r2   r   r   r   r   Zadd_bias_kvr   r   r   r   r   r   factory_kwargsr3   r5   r6   r1     s<    

zMultiheadAttention.__init__c                 C   s   | j rt| j nt| j t| j t| j | jd k	rTt| jd t| jj	d | j
d k	rht| j
 | jd k	r|t| j d S )Nr`   )r   r	   r   r   r   r   r   r   r   r   r   r   r   rq   r5   r5   r6   r     s    






z$MultiheadAttention._reset_parametersc                    s$   d|krd|d< t t| | d S )Nr   T)r0   r#   __setstate__r2   stater3   r5   r6   r     s    zMultiheadAttention.__setstate__)querykeyr,   key_padding_maskneed_weights	attn_maskaverage_attn_weightsr.   c                 C   s  |  dk}|dk	r6|j}	|	tjkr6t|s6tdd}
|sPd|   }
n6||k	s`||k	rhd}
n| jdk	r|j| jjkrd|j d| jj d	}
n| jdk	r|j| jjkrd|j d
| jj d	}
n| jrd}
n| j	sd}
n| j
dk	rd}
n| jdk	rd}
n| jrd| j d}
nh| jr,d}
nZ| js:d}
nL|dk	rJd}
n<|jrb|dk	rbd}
n$| jd dkrxd}
nt rd}
|
s\|||| j| j| jj| jjf}tj|rd}
n<tdd |D sd}
n"t rtdd |D rd}
|
s\t|||| j| j| j| j| jj| jj|dk	r4|n||||dk	rHdn|dk	rVdndS |jpp|jpp|j}|rtd d!|
  | j	r|r||kr||kr|dd } }}nd"d ||fD \}}|}nd#d |||fD \}}}| js\tj|||| j| j| j| j| j
| j| j| j| jj| jj| j|||d$| j| j | j!|d%\}}nLtj|||| j| j| j| j| j
| j| j| j| jj| jj| j||||d&\}}| j	r|r|dd|fS ||fS dS )'a
  
    Args:
        query: Query embeddings of shape :math:`(L, E_q)` for unbatched input, :math:`(L, N, E_q)` when ``batch_first=False``
            or :math:`(N, L, E_q)` when ``batch_first=True``, where :math:`L` is the target sequence length,
            :math:`N` is the batch size, and :math:`E_q` is the query embedding dimension ``embed_dim``.
            Queries are compared against key-value pairs to produce the output.
            See "Attention Is All You Need" for more details.
        key: Key embeddings of shape :math:`(S, E_k)` for unbatched input, :math:`(S, N, E_k)` when ``batch_first=False``
            or :math:`(N, S, E_k)` when ``batch_first=True``, where :math:`S` is the source sequence length,
            :math:`N` is the batch size, and :math:`E_k` is the key embedding dimension ``kdim``.
            See "Attention Is All You Need" for more details.
        value: Value embeddings of shape :math:`(S, E_v)` for unbatched input, :math:`(S, N, E_v)` when
            ``batch_first=False`` or :math:`(N, S, E_v)` when ``batch_first=True``, where :math:`S` is the source
            sequence length, :math:`N` is the batch size, and :math:`E_v` is the value embedding dimension ``vdim``.
            See "Attention Is All You Need" for more details.
        key_padding_mask: If specified, a mask of shape :math:`(N, S)` indicating which elements within ``key``
            to ignore for the purpose of attention (i.e. treat as "padding"). For unbatched `query`, shape should be :math:`(S)`.
            Binary and byte masks are supported.
            For a binary mask, a ``True`` value indicates that the corresponding ``key`` value will be ignored for
            the purpose of attention. For a float mask, it will be directly added to the corresponding ``key`` value.
        need_weights: If specified, returns ``attn_output_weights`` in addition to ``attn_outputs``.
            Default: ``True``.
        attn_mask: If specified, a 2D or 3D mask preventing attention to certain positions. Must be of shape
            :math:`(L, S)` or :math:`(N\cdot\text{num\_heads}, L, S)`, where :math:`N` is the batch size,
            :math:`L` is the target sequence length, and :math:`S` is the source sequence length. A 2D mask will be
            broadcasted across the batch while a 3D mask allows for a different mask for each entry in the batch.
            Binary, byte, and float masks are supported. For a binary mask, a ``True`` value indicates that the
            corresponding position is not allowed to attend. For a byte mask, a non-zero value indicates that the
            corresponding position is not allowed to attend. For a float mask, the mask values will be added to
            the attention weight.
        average_attn_weights: If true, indicates that the returned ``attn_weights`` should be averaged across
            heads. Otherwise, ``attn_weights`` are provided separately per head. Note that this flag only has an
            effect when ``need_weights=True``. Default: ``True`` (i.e. average weights across heads)

    Outputs:
        - **attn_output** - Attention outputs of shape :math:`(L, E)` when input is unbatched,
          :math:`(L, N, E)` when ``batch_first=False`` or :math:`(N, L, E)` when ``batch_first=True``,
          where :math:`L` is the target sequence length, :math:`N` is the batch size, and :math:`E` is the
          embedding dimension ``embed_dim``.
        - **attn_output_weights** - Only returned when ``need_weights=True``. If ``average_attn_weights=True``,
          returns attention weights averaged across heads of shape :math:`(L, S)` when input is unbatched or
          :math:`(N, L, S)`, where :math:`N` is the batch size, :math:`L` is the target sequence length, and
          :math:`S` is the source sequence length. If ``average_attn_weights=False``, returns attention weights per
          head of shape :math:`(\text{num\_heads}, L, S)` when input is unbatched or :math:`(N, \text{num\_heads}, L, S)`.

        .. note::
            `batch_first` argument is ignored for unbatched inputs.
        r   Nz>only bool and floating types of key_padding_mask are supportedr=   z5input not batched; expected query.dim() of 3 but got zKnon-self attention was used (query, key, and value are not the same Tensor)zdtypes of query (z) and self.in_proj_bias (z) don't matchz) and self.in_proj_weight (ztraining is enabledzbatch_first was not Truezself.bias_k was not Nonezself.bias_v was not Nonezdropout was z, required zerozadd_zero_attn was enabledz _qkv_same_embed_dim was not Truezattn_mask was not Nonez9key_padding_mask is not supported with NestedTensor inputr   r   znum_heads is oddzautocast is enabledz'some Tensor argument has_torch_functionc                 S   s(   g | ] }|d kp"|j p"dt|jkqS )Ncpu)Zis_cudarQ   r   .0xr5   r5   r6   
<listcomp>_  s     z.MultiheadAttention.forward.<locals>.<listcomp>z,some Tensor argument is neither CUDA nor CPUc                 S   s   g | ]}|d k	o|j qS r/   )Zrequires_gradr   r5   r5   r6   r   a  s     zhgrad is enabled and at least one of query or the input/output projection weights or biases requires_gradr   zKMultiheadAttention does not support NestedTensor outside of its fast path. z"The fast path was not hit because c                 S   s   g | ]}| d dqS r   r   	transposer   r5   r5   r6   r   ~  s     c                 S   s   g | ]}| d dqS r   r   r   r5   r5   r6   r     s     T)	rV   r   r   r   Zuse_separate_proj_weightr   r   r   r   )rV   r   r   r   r   )"rl   r   ra   rI   Zis_floating_pointr_   r   r   rV   r   r   r   r   r   r   Z	is_nestedr   Zis_autocast_enabledr   weightr   Z	overridesZhas_torch_functionallZis_grad_enabledanyZ_native_multi_head_attentionr   r   r9   Zmulti_head_attention_forwardr   r   r   )r2   r   r   r,   r   r   r   r   Z
is_batchedZ
_kpm_dtypeZwhy_not_fast_pathZtensor_argsZ
any_nestedZattn_outputZattn_output_weightsr5   r5   r6   r;     s    3





                        
zMultiheadAttention.forward)	r`   TFFNNFNN)NTNT)rB   rC   rD   rE   rF   r   ra   r   rH   r1   r   r   rI   r   r;   rJ   r5   r5   r3   r6   r#   w  s2   
7          *       c                       sZ   e Zd ZU dZdgZeed< deedd fddZe	e	d	d
dZ
edddZ  ZS )r$   a  Applies the element-wise function:

    .. math::
        \text{PReLU}(x) = \max(0,x) + a * \min(0,x)

    or

    .. math::
        \text{PReLU}(x) =
        \begin{cases}
        x, & \text{ if } x \geq 0 \\
        ax, & \text{ otherwise }
        \end{cases}

    Here :math:`a` is a learnable parameter. When called without arguments, `nn.PReLU()` uses a single
    parameter :math:`a` across all input channels. If called with `nn.PReLU(nChannels)`,
    a separate :math:`a` is used for each input channel.


    .. note::
        weight decay should not be used when learning :math:`a` for good performance.

    .. note::
        Channel dim is the 2nd dim of input. When input has dims < 2, then there is
        no channel dim and the number of channels = 1.

    Args:
        num_parameters (int): number of :math:`a` to learn.
            Although it takes an int as input, there is only two values are legitimate:
            1, or the number of channels at input. Default: 1
        init (float): the initial value of :math:`a`. Default: 0.25

    Shape:
        - Input: :math:`( *)` where `*` means, any number of additional
          dimensions.
        - Output: :math:`(*)`, same shape as the input.

    Attributes:
        weight (Tensor): the learnable weights of shape (:attr:`num_parameters`).

    .. image:: ../scripts/activation_images/PReLU.png

    Examples::

        >>> m = nn.PReLU()
        >>> input = torch.randn(2)
        >>> output = m(input)
    num_parametersr         ?N)r   initr.   c                    s<   ||d}|| _ tt|   ttj|f||| _d S )Nr   )	r   r0   r$   r1   r
   ra   r   Zfill_r   )r2   r   r   r   r   r   r3   r5   r6   r1     s    
zPReLU.__init__r7   c                 C   s   t || jS r/   )r9   Zprelur   r:   r5   r5   r6   r;     s    zPReLU.forwardrN   c                 C   s   d | jS )Nznum_parameters={})r>   r   rq   r5   r5   r6   r@     s    zPReLU.extra_repr)r   r   NN)rB   rC   rD   rE   rF   rs   rH   rG   r1   r   r;   rQ   r@   rJ   r5   r5   r3   r6   r$     s   
0    c                   @   s    e Zd ZdZeedddZdS )r%   a  Applies the element-wise function:

    .. math::
        \text{SoftSign}(x) = \frac{x}{ 1 + |x|}

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Softsign.png

    Examples::

        >>> m = nn.Softsign()
        >>> input = torch.randn(2)
        >>> output = m(input)
    r7   c                 C   s
   t |S r/   )r9   Zsoftsignr:   r5   r5   r6   r;     s    zSoftsign.forwardNrb   r5   r5   r5   r6   r%     s   c                   @   s    e Zd ZdZeedddZdS )r&   a  Applies the element-wise function:

    .. math::
        \text{Tanhshrink}(x) = x - \tanh(x)

    Shape:
        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
        - Output: :math:`(*)`, same shape as the input.

    .. image:: ../scripts/activation_images/Tanhshrink.png

    Examples::

        >>> m = nn.Tanhshrink()
        >>> input = torch.randn(2)
        >>> output = m(input)
    r7   c                 C   s
   t |S r/   )r9   Z
tanhshrinkr:   r5   r5   r6   r;     s    zTanhshrink.forwardNrb   r5   r5   r5   r6   r&     s   c                       sf   e Zd ZU dZdgZee ed< dee dd fddZ fddZ	e
e
d	d
dZdd Z  ZS )r'   a9  Applies the Softmin function to an n-dimensional input Tensor
    rescaling them so that the elements of the n-dimensional output Tensor
    lie in the range `[0, 1]` and sum to 1.

    Softmin is defined as:

    .. math::
        \text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)}

    Shape:
        - Input: :math:`(*)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(*)`, same shape as the input

    Args:
        dim (int): A dimension along which Softmin will be computed (so every slice
            along dim will sum to 1).

    Returns:
        a Tensor of the same dimension and shape as the input, with
        values in the range [0, 1]

    Examples::

        >>> m = nn.Softmin(dim=1)
        >>> input = torch.randn(2, 3)
        >>> output = m(input)
    rl   Nrn   c                    s   t t|   || _d S r/   )r0   r'   r1   rl   ro   r3   r5   r6   r1   ,  s    zSoftmin.__init__c                    s    t  | t| dsd | _d S Nrl   r0   r   hasattrrl   r   r3   r5   r6   r   0  s    
zSoftmin.__setstate__r7   c                 C   s   t j|| jddS N   Z_stacklevel)r9   Zsoftminrl   r:   r5   r5   r6   r;   5  s    zSoftmin.forwardc                 C   s   dj | jdS Nz	dim={dim})rl   rp   rq   r5   r5   r6   r@   8  s    zSoftmin.extra_repr)NrB   rC   rD   rE   rF   r   rs   rH   r1   r   r   r;   r@   rJ   r5   r5   r3   r6   r'     s   
c                       sl   e Zd ZU dZdgZee ed< dee dd fddZ fddZ	e
e
d	d
dZedddZ  ZS )r(   a  Applies the Softmax function to an n-dimensional input Tensor
    rescaling them so that the elements of the n-dimensional output Tensor
    lie in the range [0,1] and sum to 1.

    Softmax is defined as:

    .. math::
        \text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}

    When the input Tensor is a sparse tensor then the unspecifed
    values are treated as ``-inf``.

    Shape:
        - Input: :math:`(*)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(*)`, same shape as the input

    Returns:
        a Tensor of the same dimension and shape as the input with
        values in the range [0, 1]

    Args:
        dim (int): A dimension along which Softmax will be computed (so every slice
            along dim will sum to 1).

    .. note::
        This module doesn't work directly with NLLLoss,
        which expects the Log to be computed between the Softmax and itself.
        Use `LogSoftmax` instead (it's faster and has better numerical properties).

    Examples::

        >>> m = nn.Softmax(dim=1)
        >>> input = torch.randn(2, 3)
        >>> output = m(input)

    rl   Nrn   c                    s   t t|   || _d S r/   )r0   r(   r1   rl   ro   r3   r5   r6   r1   d  s    zSoftmax.__init__c                    s    t  | t| dsd | _d S r   r   r   r3   r5   r6   r   h  s    
zSoftmax.__setstate__r7   c                 C   s   t j|| jddS r   )r9   softmaxrl   r:   r5   r5   r6   r;   m  s    zSoftmax.forwardrN   c                 C   s   dj | jdS r   rp   rq   r5   r5   r6   r@   p  s    zSoftmax.extra_repr)N)rB   rC   rD   rE   rF   r   rs   rH   r1   r   r   r;   rQ   r@   rJ   r5   r5   r3   r6   r(   ;  s   
%c                   @   s    e Zd ZdZeedddZdS )r)   a|  Applies SoftMax over features to each spatial location.

    When given an image of ``Channels x Height x Width``, it will
    apply `Softmax` to each location :math:`(Channels, h_i, w_j)`

    Shape:
        - Input: :math:`(N, C, H, W)` or :math:`(C, H, W)`.
        - Output: :math:`(N, C, H, W)` or :math:`(C, H, W)` (same shape as input)

    Returns:
        a Tensor of the same dimension and shape as the input with
        values in the range [0, 1]

    Examples::

        >>> m = nn.Softmax2d()
        >>> # you softmax over the 2nd dimension
        >>> input = torch.randn(2, 3, 12, 13)
        >>> output = m(input)
    r7   c                 C   s0   |  dks |  dks tdtj|dddS )N   r   z-Softmax2d requires a 3D or 4D tensor as inputr   r   )rl   r_   r9   r   r:   r5   r5   r6   r;     s     zSoftmax2d.forwardNrb   r5   r5   r5   r6   r)   t  s   c                       sf   e Zd ZU dZdgZee ed< dee dd fddZ fddZ	e
e
d	d
dZdd Z  ZS )r*   a  Applies the :math:`\log(\text{Softmax}(x))` function to an n-dimensional
    input Tensor. The LogSoftmax formulation can be simplified as:

    .. math::
        \text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right)

    Shape:
        - Input: :math:`(*)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(*)`, same shape as the input

    Args:
        dim (int): A dimension along which LogSoftmax will be computed.

    Returns:
        a Tensor of the same dimension and shape as the input with
        values in the range [-inf, 0)

    Examples::

        >>> m = nn.LogSoftmax(dim=1)
        >>> input = torch.randn(2, 3)
        >>> output = m(input)
    rl   Nrn   c                    s   t t|   || _d S r/   )r0   r*   r1   rl   ro   r3   r5   r6   r1     s    zLogSoftmax.__init__c                    s    t  | t| dsd | _d S r   r   r   r3   r5   r6   r     s    
zLogSoftmax.__setstate__r7   c                 C   s   t j|| jddS r   )r9   Zlog_softmaxrl   r:   r5   r5   r6   r;     s    zLogSoftmax.forwardc                 C   s   dj | jdS r   rp   rq   r5   r5   r6   r@     s    zLogSoftmax.extra_repr)Nr   r5   r5   r3   r6   r*     s   
)1r]   typingr   r   ra   r   Zlinearr   Ztorch.nn.initr   r   r	   Ztorch.nn.parameterr
   moduler   r=   r   r9   __all__r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r5   r5   r5   r6   <module>   s|                     2*AE')$(.,1!)*2*(  'B/9