U <ºc¨ã@s6ddlZddlmZddlmZGdd„dejƒZdS)éN)Ú ShardedTensorcsLeZdZdejf‡fdd„ Zdd„Zdd„Zdd „Zd d „Z d d „Z ‡Z S)ÚSimpleMegatronLMNcsdtƒ ¡tj|dd|iŽ|_t ¡|_tj|dd|iŽ|_|dk r`|j |¡|j |¡dS)NrÚdtypeé) ÚsuperÚ__init__ÚnnZLinearÚfc1ZGELUÚgeluÚfc2Zcuda)ÚselfZ linear_sizeZrankr©Ú __class__©úZ/tmp/pip-unpacked-wheel-gikjz4vx/torch/testing/_internal/distributed/_shard/test_common.pyrs   zSimpleMegatronLM.__init__cCs| | | |¡¡¡S©N)r r r )r ÚinprrrÚforwardszSimpleMegatronLM.forwardcCsPt|jjtƒr|jj ¡}n|jj}t|jjtƒr@|jj ¡}n|jj}||fSr)Ú isinstancer ÚweightrZ local_tensorr )r Zweight1Zweight2rrrÚ get_weightsszSimpleMegatronLM.get_weightscCs|jj|jjfSr)r Úbiasr ©r rrrÚ get_biases!szSimpleMegatronLM.get_biasescCs|jjj|jjjfSr)r rÚgradr rrrrÚget_weight_grads$sz!SimpleMegatronLM.get_weight_gradscCs|jjj|jjjfSr)r rrr rrrrÚget_bias_grads'szSimpleMegatronLM.get_bias_grads) Ú__name__Ú __module__Ú __qualname__ÚtorchZfloat32rrrrrrÚ __classcell__rrr rrs   r)r Ztorch.nnrZ'torch.distributed._shard.sharded_tensorrÚModulerrrrrÚs