U
    <ºc_#  ã                   @   s  d dl Z d dlZd dlmZ ddlmZ d dlmZmZ ddgZG dd„ deƒZ	dee ee ee ee ee ee e
e
eeeeed
œdd„Zee ee ee ee ee ee eeeeee
dœdd„Zee ee ee ee ee ee eeeeee
dœdd„ZdS )é    N)ÚTensoré   )Ú	Optimizer)ÚListÚOptionalÚASGDÚasgdc                       sL   e Zd ZdZdee ed	œ‡ fd
d„Z‡ fdd„Ze 	¡ ddd„ƒZ
‡  ZS )r   aÊ  Implements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        foreach (bool, optional): whether foreach implementation of optimizer
            is used (default: None)
        maximize (bool, optional): maximize the params based on the objective, instead of
            minimizing (default: False)

    .. _Acceleration of stochastic approximation by averaging:
        https://dl.acm.org/citation.cfm?id=131098
    ç{®Gáz„?ç-Cëâ6?ç      è?ç    €„.Ar   NF)ÚforeachÚmaximizec	           
   	      sX   d|kst d |¡ƒ‚d|ks,t d |¡ƒ‚t|||||||d}	tt| ƒ ||	¡ d S )Nç        zInvalid learning rate: {}zInvalid weight_decay value: {})ÚlrÚlambdÚalphaÚt0Úweight_decayr   r   )Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)
ÚselfÚparamsr   r   r   r   r   r   r   Údefaults©Ú	__class__© ú4/tmp/pip-unpacked-wheel-gikjz4vx/torch/optim/asgd.pyr   !   s    
  ÿzASGD.__init__c                    s  t ƒ  |¡ | jD ]}| dd ¡ | dd¡ qt| j ¡ ƒ}t|ƒdkoZt 	|d d ¡}|s€|D ]}t 
t|d ƒ¡|d< qdt|ƒdkoœt 	|d d ¡}|s¾|D ]}t 
|d ¡|d< q¦t|ƒdkoÚt 	|d d ¡}|s|D ]}t 
t|d ƒ¡|d< qæd S )Nr   r   Fr   ÚstepÚetaÚmu)r   Ú__setstate__Úparam_groupsÚ
setdefaultÚlistÚstateÚvaluesÚlenÚtorchZ	is_tensorÚtensorÚfloat)r   r(   ÚgroupZstate_valuesZstep_is_tensorÚsZeta_is_tensorZmu_is_tensorr   r   r    r$   ,   s"    
zASGD.__setstate__c                 C   s\  d}|dk	r&t  ¡  |ƒ }W 5 Q R X | jD ](}g }g }g }g }g }g }	|d D ]Ä}
|
jdk	rR| |
¡ |
jjrztdƒ‚| |
j¡ | j|
 }t|ƒdkrÞt  	d¡|d< t  	|d ¡|d< t  	d	¡|d
< t j
|
t jd|d< | |d
 ¡ | |d ¡ | |d ¡ |	 |d ¡ qRt||||||	|d |d |d |d |d |d |d d q,|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   z&ASGD does not support sparse gradientsr   r   r!   r   r"   g      ð?r#   )Zmemory_formatÚaxr   r   r   r   r   r   )r   r   r   r   r   r   r   )r+   Zenable_gradr%   ÚgradÚappendZ	is_sparseÚRuntimeErrorr(   r*   r,   Z
zeros_likeZpreserve_formatr   )r   ÚclosureZlossr.   Zparams_with_gradÚgradsÚmusÚaxsÚetasÚstate_stepsÚpr(   r   r   r    r!   ?   sT    



ôz	ASGD.step)r	   r
   r   r   r   NF)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úboolr   r$   r+   Zno_gradr!   Ú__classcell__r   r   r   r    r   
   s       ÿ ÿF)r   r5   r7   r6   r8   r9   r   r   r   r   r   r   r   c                C   s^   |dkrd}|r"t j ¡ r"tdƒ‚|r6t j ¡ s6t}nt}|| |||||||	|
|||d dS )znFunctional API that performs asgd algorithm computation.

    See :class:`~torch.optim.ASGD` for details.
    NFz6torch.jit.script not supported with foreach optimizers)r   r   r   r   r   r   )r+   ZjitZis_scriptingr3   Ú_multi_tensor_asgdÚ_single_tensor_asgd)r   r5   r7   r6   r8   r9   r   r   r   r   r   r   r   Úfuncr   r   r    r   y   s(    õ)r   r5   r7   r6   r8   r9   r   r   r   r   r   r   c             	   C   sB  t | ƒD ]2\}}|| }|s"|n| }|| }|| }|| }|| }t |¡rpt |¡}t |¡}t |¡}|d7 }| ¡ }|
dkr–|j||
d}| d|| ¡   ¡ |j|| ¡  d | ¡ dkrä| | |¡ 	|¡¡ n
| 
|¡ t |t d|| |  |	¡ ¡}| 
|¡ t dtd|| ƒ ¡}| 
|¡ qd S )Nr   r   ©r   )Ú	enumerater+   Ú
is_complexÚview_as_realÚitemÚaddZmul_Úadd_ÚsubÚmulÚcopy_r,   ÚmathÚpowÚmax)r   r5   r7   r6   r8   r9   r   r   r   r   r   r   ÚiÚparamr1   r#   r0   r"   Zstep_tr!   Únew_etaÚnew_mur   r   r    rB   ¨   s0    




"
rB   c             
   C   sl  t | ƒdkrd S |rt |¡}dd„ }||ƒ}|| ƒ} ||ƒ}t |d¡ |
dkrbtj|| |
d}|d  ¡ }t | d||  ¡ tj| || d tt |ƒƒD ]N}||  ¡ dkrÜ||  | |  	|| ¡ 
|| ¡¡ q ||  | | ¡ q tt |ƒƒD ]j}t |t d|| ||  ¡   |	¡ ¡}||  |¡ t dtd||  ¡ | ƒ ¡}||  |¡ qüd S )Nr   c                 S   s   dd„ | D ƒS )Nc                 S   s$   g | ]}t  |¡rt  |¡n|‘qS r   )r+   rF   rG   )Ú.0Útr   r   r    Ú
<listcomp>ñ   s     zE_multi_tensor_asgd.<locals>._view_complex_as_real.<locals>.<listcomp>r   )Ztensor_listr   r   r    Ú_view_complex_as_realð   s    z1_multi_tensor_asgd.<locals>._view_complex_as_realr   rD   )r*   r+   Z_foreach_negZ_foreach_add_Z_foreach_addrH   Z_foreach_mul_ÚrangerJ   rK   rL   rM   r,   rN   rO   rP   )r   r5   r7   r6   r8   r9   r   r   r   r   r   r   rX   r"   rQ   rS   rT   r   r   r    rA   Ü   s.    
(* rA   )NF)rN   r+   r   Z	optimizerr   Útypingr   r   Ú__all__r   r?   r-   r   rB   rA   r   r   r   r    Ú<module>   s`   w  ÷ñ/ô4ô