U
    Jc                     @   sx   d Z ddlmZmZ ddlZddlmZmZ ddlmZ ddl	m
Z
 ddlmZ d	gZed
ejdZG dd	 d	e
ZdS )zDTracks the running statistics per mini-batch instead of micro-batch.    )TypeVarcastN)Tensornn)
batch_norm)
_BatchNorm   )is_recomputingDeferredBatchNormTModule)boundc                       s   e Zd ZU dZeed< eed< eed< eed< eed< deeeeedd fddZ	eddddZ
eedddZddddZeedddZedeeedddZ  ZS )r
   zeA BatchNorm layer tracks multiple micro-batches to update running
    statistics per mini-batch.
    sumsum_squaresrunning_meanrunning_varnum_batches_trackedh㈵>皙?Tr   N)num_featuresepsmomentumaffinechunksreturnc                    sT   t  j||||dd | dt| j | dt| j d| _d| _|| _	d S )NT)track_running_statsr   r   r   )
super__init__register_buffertorchZ
zeros_liker   r   countertrackedr   )selfr   r   r   r   r   	__class__ M/tmp/pip-unpacked-wheel-gikjz4vx/torch/distributed/pipeline/sync/batchnorm.pyr   "   s    zDeferredBatchNorm.__init__)inputr   c                 C   s    |  dkrtd|   d S )N   z*expected at least 3D input (got %dD input))dim
ValueError)r!   r&   r$   r$   r%   _check_input_dim3   s    z"DeferredBatchNorm._check_input_dimc              	   C   s   dg}| td|  t 2 |  j||7  _|  j|d |7  _W 5 Q R X |  |d }|  j	|7  _	|  j
d7  _
| j
| jkS )z#Tracks statistics of a micro-batch.r   r'   r   )extendranger(   r   Zno_gradr   r   sizeZnumelr   r    r   )r!   r&   r(   r-   r$   r$   r%   _track8   s    
"zDeferredBatchNorm._track)r   c                 C   s   d}|  j d7  _ | jdkr,dt| j  }n| j}| j| j }| j| j |d  }|}|  jd| 9  _|  j|| 7  _|  jd| 9  _|  j|| 7  _| j  | j  d| _d| _	dS )z/Updates the running statistics of a mini-batch.        r   Ng      ?r'   r   )
r   r   floatr   r   r   r   r   Zzero_r    )r!   Zexponential_average_factorZmeanvarmr$   r$   r%   _commitH   s     


zDeferredBatchNorm._commitc              
   C   sb   | j s(t|| j| j| j| jdd| jdS t sD| |}|rD| 	  t|d d | j| jdd| jdS )NFr/   )r   r   weightbiastrainingr   r   T)
r6   r   r   r   r4   r5   r   r	   r.   r3   )r!   r&   Ztracked_enoughr$   r$   r%   forwardb   s2    
zDeferredBatchNorm.forward)moduler   r   c                 C   s   t |tr|j|krtt|S |}t |tr|jrt|j|j|j	|j
|}|j
rl|d|j |d|j |d|j |d|j |d|j | D ]\}}||| || qtt|S )a]  Converts a :class:`nn.BatchNorm` or underlying
        :class:`nn.BatchNorm`s into :class:`DeferredBatchNorm`::

            from torchvision.models.resnet import resnet101
            from torchpipe.batchnorm import DeferredBatchNorm
            model = resnet101()
            model = DeferredBatchNorm.convert_deferred_batch_norm(model)

        r4   r5   r   r   r   )
isinstancer
   r   r   r   r   r   r   r   r   r   Zregister_parameterr4   r5   r   r   r   r   Znamed_childrenZ
add_moduleconvert_deferred_batch_norm)clsr8   r   Zmodule_outputnamechildr$   r$   r%   r:      s    
z-DeferredBatchNorm.convert_deferred_batch_norm)r   r   Tr   )r   )__name__
__module____qualname____doc__r   __annotations__intr0   boolr   r*   r.   r3   r7   classmethodr   r:   __classcell__r$   r$   r"   r%   r
      s0   
    $)rA   typingr   r   r   r   r   Ztorch.nn.functionalr   Ztorch.nn.modules.batchnormr   
checkpointr	   __all__Moduler   r
   r$   r$   r$   r%   <module>   s   