U Jc|@shUddlmZmZmZddlZddlmmZddlm Z gZ ee e d<ej jGdddeZdS))ListOptionalDictN)Tensor__all__c @sZeZdZdeeeeeeeeeed ddZeeeddd Z eeed d d Z d S)_FunctionalSGD{Gz?F) paramslrmomentum dampening weight_decaynesterovmaximizeforeach_allow_empty_param_listc Csl||||d|_||_||_||_tjttjtt tjffi|_ t |dkr^| s^t dd|i|_ dS)N)r r r rrz%optimizer got an empty parameter listr )defaultsrrrtorchjitZannotaterrstrstatelen ValueError param_group) selfr r r r rrrrrrJ/tmp/pip-unpacked-wheel-gikjz4vx/torch/distributed/optim/functional_sgd.py__init__s $z_FunctionalSGD.__init__)paramgradc Cs|jd}|jd}|jd}|jd}|g}g}g} d} |dk r| ||jrVd} ||jkrji|j|<|j|} d| kr|dn|| dt,tj|| ||||||j|j | |j d W5QRX|j|} |d } | dk r| | d<dS) z` Similar to self.step, but operates on a single parameter and its gradient. rr r r FNTmomentum_bufferrr r r rrhas_sparse_gradrr) rappend is_sparserrno_gradFsgdrrr) rrr rr r r r momentum_buffer_listgradsr#rr!rrr step_param2sH           z_FunctionalSGD.step_param) gradientscCsh|jd}g}g}g}|jd}|jd}|jd}|jd} t|t|krttddt|dd t|d } t||D]n\} } | dk r|| || | jrd } | |jkri|j| <|j| } d | kr|dq|| d qt ,t j ||||||| |j |j | |jd W5QRXt|D].\}}|j|} ||}|dk r4|| d <q4dS)Nr r rr r zEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: FTr!r")rrrrzipr$r%rrr&r'r(rrr enumerate)rr,r Zparams_with_gradr*r)r rr r r#rZgradientripr!rrrstepas^               z_FunctionalSGD.stepN)rr r r FFFF) __name__ __module__ __qualname__rrfloatboolrrr+r1rrrrrs* /r)typingrrrrZtorch.optim._functionalZoptimZ _functionalr'rrr__annotations__rscriptobjectrrrrrs