U
    <ºcj)  ã                   @   sÂ   d dl Z d dlZd dlm  mZ d dlm  mZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ G dd„ dƒZG dd	„ d	ejƒZG d
d„ dejƒZdd„ Zdd„ Zdd„ ZG dd„ deƒZdS )é    N)Úoptim)ÚDistributedOptimizer)Ú	dist_init)ÚRpcAgentTestFixturec                   @   s.   e Zd Ze ¡ Zd	dd„Zdd„ Zdd„ ZdS )
ÚMyModuleTc                 C   s(   t  ¡ }| d¡ t jd||d| _d S )Nr   ©é   r   ©Úrequires_gradÚ	generator)ÚtorchÚ	GeneratorÚmanual_seedÚrandÚw)Úselfr
   Úg_cpu© r   ú_/tmp/pip-unpacked-wheel-gikjz4vx/torch/testing/_internal/distributed/rpc/dist_optimizer_test.pyÚ__init__   s    
zMyModule.__init__c                 C   s   t  | j|¡S ©N)r   Úmmr   )r   Út1r   r   r   Úforward   s    zMyModule.forwardc                 C   s   | j S r   )r   ©r   r   r   r   Úget_w   s    zMyModule.get_wN)T)	Ú__name__Ú
__module__Ú__qualname__Ú	threadingÚLockÚlockr   r   r   r   r   r   r   r      s   
	r   c                       s&   e Zd Z‡ fdd„Zddd„Z‡  ZS )ÚFailingOptimizerc                    s   t ƒ  |i ¡ d S r   )Úsuperr   ©r   Úparams©Ú	__class__r   r   r   #   s    zFailingOptimizer.__init__Nc                 C   s   t dƒ‚d S )NzError running optimizer.)Ú
ValueError©r   Úclosurer   r   r   Ústep&   s    zFailingOptimizer.step)N©r   r   r   r   r+   Ú__classcell__r   r   r&   r   r"   "   s   r"   c                       s&   e Zd Z‡ fdd„Zddd„Z‡  ZS )ÚOptimizerFailingOnConstructorc                    s   t ƒ  |i ¡ tdƒ‚d S )NúError creating optimizer.)r#   r   r(   r$   r&   r   r   r   +   s    z&OptimizerFailingOnConstructor.__init__Nc                 C   s   t ‚d S r   )ÚNotImplementedErrorr)   r   r   r   r+   /   s    z"OptimizerFailingOnConstructor.step)Nr,   r   r   r&   r   r.   *   s   r.   c                 O   s   | |  ¡ f|ž|ŽS r   )Zlocal_value©ÚmethodZobj_rrefÚargsÚkwargsr   r   r   Ú_call_method3   s    r5   c                 O   s"   t j| ¡ t| |gt|ƒ |dS )aZ  
    Call rpc.remote on a method in a remote object.

    Args:
        method: the method (for example, Class.method)
        obj_rref (RRef): remote reference to the object
        args: positional arguments to pass to the method
        kwargs: keyword arguments to pass to the method

    Returns a RRef to the remote method call result.
    ©r3   r4   )ÚrpcÚremoteÚownerr5   Úlistr1   r   r   r   Úremote_method7   s    ür;   c                 O   s"   t j| ¡ t| |gt|ƒ |dS )aX  
    Call rpc.rpc_async on a method in a remote object.

    Args:
        method: the method (for example, Class.method)
        obj_rref (RRef): remote reference to the object
        args: positional arguments to pass to the method
        kwargs: keyword arguments to pass to the method

    Returns a Future to the method call result.
    r6   )r7   Ú	rpc_asyncr9   r5   r:   r1   r   r   r   Úrpc_async_methodK   s    ür=   c                   @   sT   e Zd Zeƒ dd„ ƒZeƒ dd„ ƒZdd„ Zeƒ dd„ ƒZd	d
„ Zeƒ dd„ ƒZ	dS )ÚDistOptimizerTestc              
   C   s  d| j d | j  }d| j d | j  }t |t¡}t |t¡}ttj|ƒ}ttj|ƒ}tt||gƒ}t	 
¡ œ}t ¡ }	|	 d¡ tjdd|	d}
tjdd|	d}ttj||ƒ}ttj|| ¡ ƒ}t | ¡ |
¡ ¡ }t	 ||g¡ |  td¡ | |¡ W 5 Q R X W 5 Q R X d S )	Núworker%dé   é   r   r   Tr	   zError running optimizer)ÚrankÚ
world_sizer7   r8   r   r;   r   r   r"   Údist_autogradÚcontextr   r   r   r   r=   r   ÚwaitÚaddÚsumÚbackwardÚassertRaisesRegexÚ	Exceptionr+   )r   Úowner1Úowner2Úremote_module1Úremote_module2Úremote_param1Úremote_param2Ú
dist_optimÚ
context_idr   r   Út2Úoutput1Úoutput2Úlossr   r   r   Útest_dist_optim_exception`   s*     ÿ

z+DistOptimizerTest.test_dist_optim_exceptionc              	   C   s‚   d| j d | j  }d| j d | j  }t |t¡}t |t¡}ttj|ƒ}ttj|ƒ}|  td¡ t	t
||gƒ}W 5 Q R X d S )Nr?   r@   rA   r/   )rB   rC   r7   r8   r   r;   r   rJ   rK   r   r.   )r   rL   rM   rN   rO   rP   rQ   rR   r   r   r   Ú(test_dist_optim_exception_on_constructor|   s     ÿz:DistOptimizerTest.test_dist_optim_exception_on_constructorc              	   O   s*  t ƒ }t ƒ }| ¡ | ¡ g}||f|ž|Ž}|j ¡  ¡ }|j ¡  ¡ }	t ¡ }
|
 d¡ tjdd|
d}tjdd|
d}| 	|¡}| 	|¡}t 
||¡ ¡ }| ¡  | ¡  d| jd | j  }d| jd | j  }t |t ¡}t |t ¡}tt j|ƒ}tt j|ƒ}| ¡ }|  || ¡ ¡ |  |	| ¡ ¡ t|||gf|ž|Ž}t ¡ Ô}|
 d¡ tjdd|
d}tjdd|
d}tt j	||ƒ}tt j	|| ¡ ƒ}t 
| ¡ |¡}t || ¡ g¡ | |¡ tt j|ƒ ¡ }tt j|ƒ ¡ }|  ||¡ |  |	|¡ |  || ¡ ¡ |  || ¡ ¡ W 5 Q R X d S )Nr   r   Tr	   r?   r@   rA   )r   r   r   ÚcloneÚdetachr   r   r   r   r   rG   rH   rI   r+   rB   rC   r7   r8   r;   Úto_hereÚassertEqualr   rD   rE   r=   rF   ÚassertNotEqual)r   Ú	optim_clsr3   r4   Úmodule1Úmodule2r%   Úlocal_optimÚold_w1Úold_w2r   r   rT   rU   rV   rW   rL   rM   rN   rO   rP   rQ   Zold_w1_remoterR   rS   Únew_w1Únew_w2r   r   r   Ú_test_dist_optim_baseŒ   s^    


 ÿÿÿ


z'DistOptimizerTest._test_dist_optim_basec                 C   sž   | j tjdd | j tjddd | j tjddd | j tjdd | j tjddddd | j tjd	d
 | j tjdd | j tjdd | j tj	dd d S )Nçš™™™™™©?©Úlrg{®Gáz„?T)rj   Zamsgradgü©ñÒMbP?r@   )rj   ZmomentumZweight_decayZnesterovçffffffî?©Úrho)
rg   r   ZAdagradZAdamZAdamWÚSGDÚAdadeltaÚRMSpropZAdamaxÚRpropr   r   r   r   Útest_dist_optimÉ   s    z!DistOptimizerTest.test_dist_optimc              	   O   s*  t ƒ }t dd}| ¡ | ¡ g}||f|ž|Ž}|j ¡  ¡ }|j ¡  ¡ }	t ¡ }
|
 d¡ tjdd|
d}tjdd|
d}| 	|¡}| 	|¡}t 
||¡ ¡ }| ¡  | ¡  d| jd | j  }d| jd	 | j  }t |t ¡}tj|t d
d}| ¡  ¡ }| ¡  ¡ }|  || ¡ ¡ |  |	| ¡ ¡ t|||gf|ž|Ž}t ¡ Ô}|
 d¡ tjdd|
d}tjdd|
d}| ¡  	|¡}| ¡  	| ¡ ¡}t 
| ¡ |¡}t || ¡ g¡ | |¡ | ¡  ¡  ¡ }| ¡  ¡  ¡ }|  ||¡ |  |	|¡ |  || ¡ ¡ |  || ¡ ¡ W 5 Q R X d S )NF)r
   r   r   Tr	   r?   r@   rA   )F)r3   )r   r   r   rZ   r[   r   r   r   r   r   rG   rH   rI   r+   rB   rC   r7   r8   r]   r\   r   rD   rE   r<   rF   r^   )r   r_   r3   r4   r`   ra   r%   rb   rc   rd   r   r   rT   rU   rV   rW   rL   rM   rN   rO   rP   rQ   rR   rS   re   rf   r   r   r   Ú_test_dist_optim_none_gradsÕ   s\    



 ÿÿÿ


z-DistOptimizerTest._test_dist_optim_none_gradsc                 C   sD   | j tjdd | j tjdd | j tjdd | j tjdd d S )Nrh   ri   rk   rl   )rs   r   rn   rp   rq   ro   r   r   r   r   Útest_dist_optim_none_grads  s    z,DistOptimizerTest.test_dist_optim_none_gradsN)
r   r   r   r   rX   rY   rg   rr   rs   rt   r   r   r   r   r>   _   s   

=
=r>   )r   r   Ztorch.distributed.autogradZdistributedZautogradrD   Ztorch.distributed.rpcr7   r   Ztorch.distributed.optimr   Z"torch.testing._internal.dist_utilsr   Z>torch.testing._internal.distributed.rpc.rpc_agent_test_fixturer   r   Z	Optimizerr"   r.   r5   r;   r=   r>   r   r   r   r   Ú<module>   s   	