U <c @srddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z m Z ddlmZddlmZddlmZmZmZmZddlZddlZddlZddlmZddlmmm m!Z!ddl"mmm m#Z$ddl%mmm m&Z'ddl(m)Z)ddl*m)m+Z,ddl-m)m&m.Z.ddl/m0Z1ddl/m2Z3dd l4m5Z5m6Z6dd l7m8Z9m:Z;mZ?dd l@mAZAmBZBmCZCmDZDdd lEmFZFmGZGdd lHmIZIddlJmKZKddlLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_ddl`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZiddljmmkmlZlddlmmnZnzddloZodZpWneqk rdZpYnXejrdkrddlsZsnddltZtGddde)juZvGdddZwewdZxdex_yewezddZ{ddddidexe{ddd dd!dd"dgggZ|ej}j~ej}jej}jej}jgZej}jej}jej}j~ej}jgZej}jej}jej}j~ej}jgZd#Ze d$eZGd%d&d&eZeiep d'Zejd(Zed)d*Zd+Zd,d-iZd.d/Zd0Zd1Zd2Zd3Zd4ZGd5d6d6eZGd7d8d8e)juZGd9d:d:e)juZGd;d<dd>e)juZGd?d@d@e)juZGdAdBdBe)juZGdCdDdDe)juZGdEdFdFe)juZGdGdHdHe)juZGdIdJdJe)juZeZeZeddKZe)jd dLdMZdNdOZdPZd+d"dQZdRdSZdTdUZdVdWZe dXdYZe dZd[Zdejdfd\d]Zdejfd^d_Zd`daZdbdcZGdddedeeZGdfdgdgeMZGdhdidiZeaejdS)jN) namedtuple OrderedDict)contextmanagersuppress) timedelta)reduce)Union NamedTupleCallableAny)TEST_MASTER_ADDR)TEST_MASTER_PORT) GradScalerautocast)post_localSGD_hook powerSGD_hook default_hooks quantization)get_world_size_get_default_groupAllreduceOptions GroupMember)$_verify_param_shape_across_processes_sync_module_states)DistributedDataParallel)_dump_DDP_relevant_env_vars)MultiProcessTestCase TEST_SKIPSinit_multigpu_helperinitialize_temp_directoriescleanup_temp_dirsimple_sparse_reduce_tests skip_if_rocmskip_if_small_worldsizeskip_if_odd_worldsizeskip_if_lt_x_gpunccl_skip_if_lt_x_gpuskip_if_no_gpurequire_n_gpus_for_nccl_backendrequires_nccl_versioncaptured_outputwith_nccl_blocking_waitwith_dist_debug_levelsverify_ddp_error_logged DistTestCases) instantiate_parametrized_testsIS_MACOS IS_WINDOWS FILE_SCHEMA IS_FBCODENO_MULTIPROCESSING_SPAWN parametrizesandcastle_skipsandcastle_skip_if)DistributedSamplerTFwin32cs$eZdZfddZddZZS)NetWithBufferscsFttjdddd|_tjdddd|_|dtdddSN Fbiasbuffer super__init__nnLinearabregister_buffertorchrandnself __class__X/tmp/pip-unpacked-wheel-gikjz4vx/torch/testing/_internal/distributed/distributed_test.pyrDcs zNetWithBuffers.__init__cCs|jd|||SNr?)r@Zadd_rHrGrMxrPrPrQforwardis zNetWithBuffers.forward__name__ __module__ __qualname__rDrU __classcell__rPrPrNrQr:bs r:c@seZdZddZddZdS)FoocCs ||_dSNrTrSrPrPrQrDnsz Foo.__init__cCs:dd}|jD]"\}}|j|}|||sdSqdS)NcSs t|tjrt||S||kSr\) isinstancerJTensorequal)valueotherrPrPrQeqss  zFoo.__eq__..eqFT)__dict__items)rMrbrcattrraZ other_valuerPrPrQ__eq__rs   z Foo.__eq__N)rWrXrYrDrgrPrPrPrQr[msr[r<r?nested)key1key2Zkey3ZfoorAstringrGrHr c@s"eZdZUejed<ejed<dS)TestNamedTupleInput_1rGrHN)rWrXrYrJtensor__annotations__rPrPrPrQrps  rpzno torchvisionBACKEND INIT_METHODenv://i,test_DistributedDataParallelcs.t|tjjr|n|j}fdd|DS)Ncsg|]}|jr|qSrP)nameendswith.0eventpostfixrPrQ s z'get_profiling_event..)r^rJprofilerprofileeventsZfunction_events)r~rZ event_listrPr}rQget_profiling_events  rz:Expected to have finished reduction in the prior iterationz:passing the keyword argument `find_unused_parameters=True`z.Since `find_unused_parameters=True` is enabledz:`forward` function outputs participate in calculating losszMset the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAILc@s^eZdZUeed<ejed<eej e fed<e ed<dZ e ed<dZeed<dZeed <dS) DDPUnevenTestInputrxmodelinp sync_intervalFthrow_on_early_terminationNhookstate)rWrXrYstrrrrEModulerrJrqtupleintrboolrr rr rPrPrPrQrs    rcs$eZdZfddZddZZS)_FC2cs.tt|tjdddd|_d|jj_dS)Nr<2Tr=F)rCrrDrErFfcr> requires_gradrLrNrPrQrDsz _FC2.__init__cCs||}|Sr\rrSrPrPrQrUs z _FC2.forwardrVrPrPrNrQrs rcs$eZdZfddZddZZS)Netcsftt|tjdddd|_t|_tjdddd|_t |_ tj t ddgdd|_dS)NrAr<Fr=rrir)rCrrDrErFfc1rfc2fc3ReLUrelu ParameterrJrqlongZ no_grad_paramrLrNrPrQrDs z Net.__init__cCs8|||}|||}||}tj|ddS)Nr?dim)rrrrFsoftmaxrSrPrPrQrUs z Net.forwardrVrPrPrNrQrs rcs$eZdZfddZddZZS)LargeNetcs6tt|tjdddd|_tjdddd|_dS)NiFr=rw)rCrrDrErFrrrLrNrPrQrDszLargeNet.__init__cCs||}||}|Sr\rrrSrPrPrQrUs  zLargeNet.forwardrVrPrPrNrQrs rcs$eZdZfddZddZZS)Taskcs"tttdd|_dSNrA)rCrDrErrJonesprLrNrPrQrD s z Task.__init__cCs |j|Sr\)rrSrPrPrQrUsz Task.forwardrVrPrPrNrQr s rcs&eZdZdfdd ZddZZS) BatchNormNetTcsFtt|tjdddd|_tjd|d|_tjdddd|_dS)NrA(Fr=riaffine) rCrrDrErFr BatchNorm1dbnr)rMrrNrPrQrDszBatchNormNet.__init__cCs@t||d}||}t|d}||}tj|ddS)N)rir<)rrr?r)rJreshaperrrrrrSrPrPrQrUs    zBatchNormNet.forward)TrVrPrPrNrQrsrcs$eZdZfddZddZZS)UnusedParamTwoLinLayerNetcsDttjdddd|_tjdddd|_tjdddd|_dS)Nr<Fr=rn)rCrDrErFrGrHcrLrNrPrQrD"s z"UnusedParamTwoLinLayerNet.__init__cCs||}||}||fSr\rorMrTrGrHrPrPrQrU(s  z!UnusedParamTwoLinLayerNet.forwardrVrPrPrNrQr!s rcs$eZdZfddZddZZS)DictOutputModulecstt|_dSr\)rCrDrmodulerLrNrPrQrD/s zDictOutputModule.__init__cCs(||}|d|d}||dS)Nrr?) predictionsloss)rsum)rMrTrrrPrPrQrU3s  zDictOutputModule.forwardrVrPrPrNrQr.s rcs$eZdZfddZddZZS)TwoLinLayerNetcs2ttjdddd|_tjdddd|_dS)Nr<Fr=r?)rCrDrErFrGrHrLrNrPrQrD=s zTwoLinLayerNet.__init__cCs||}||}||fSr\rorrPrPrQrUBs  zTwoLinLayerNet.forwardrVrPrPrNrQr<s rcs*eZdZdZdfdd ZddZZS)EmbeddingNetDifferentParamsz{ A module containing an embedding with different dimension or different # of parameters depending on the rank. FcsVt|s|dkrdnd}tjd|d|_t|d|_|rRtjdddd|_dS) Nrrwrr<)Znum_embeddings embedding_dimr?Fr=)rCrDrEZ Embedding embeddingrFlinlin2)rMrankdiff_num_paramsrrNrPrQrDMs  z$EmbeddingNetDifferentParams.__init__cCs||}||Sr\)rrrSrPrPrQrUUs z#EmbeddingNetDifferentParams.forward)F)rWrXrY__doc__rDrUrZrPrPrNrQrHsrcs$eZdZfddZddZZS)ControlFlowToyModelcs6tt|tjdddd|_tjdddd|_dSNr<Fr=)rCrrDrErFlin1rrLrNrPrQrD[szControlFlowToyModel.__init__cCsHt|tjdd|jd}|r4|t||St||SdS)Nr<device)rJr`rrrrrrrMrTZuse_second_layerrPrPrQrU`szControlFlowToyModel.forwardrVrPrPrNrQrZs rrGz?)momentumcCs&|dd}|tkrt|StSdS)N.r)splitCUSTOMIZED_TIMEOUTDEFAULT_TIMEOUT)Ztest_id test_namerPrPrQ get_timeoutosr<)test_ddp_uneven_inputsZ test_ddp_model_diff_across_rankscCst|krtd|SddS)Nz%Test requires backend to be one of %scSs|Sr\rPfuncrPrPrQz!require_backend..)rsr6backendsrPrPrQrequire_backends rcs2ddtfdd|Ds*td|SddS)NcSsf|tjjkrtS|tjjkr(tS|tjjkr.checkc3s|]}t|VqdSr\)rr)r{rrrPrQ sz-require_backends_available..z)Test requires backends to be available %scSs|Sr\rPrrPrPrQrrz,require_backends_available..)allr6rrPrrQrequire_backends_availables  rcCs&ttjd|krtd|SddS)N WORLD_SIZEzTest requires world size of %dcSs|Sr\rPrrPrPrQrrz$require_world_size..)rosenvironr6 world_sizerPrPrQrequire_world_sizes rc cstjd}tj|d}t|d}zBtjdkrLt| tjddVnt | t jdVW5tjdkrt| tj dnt | t j |XW5QRXdS)NTEMP_DIRlockfilewr9r?)rrpathjoinopensysplatformmsvcrtZlockingfilenoZLK_UNLCKfcntlZflockZLOCK_UNcloseZLK_RLCKZLOCK_EX)rrlfrPrPrQ_locks     rc cshtdkr$t\}}t|nd}|g}t||d}z |VW5tdkrbt|XdSNr)rget_ranktempfilemkstemprrbroadcast_object_listremove)fdrxZ object_listrPrPrQ_rank_temp_files      rcCsN|dkr |}|dkr,tj||||d|Stj||||d||SdSNdtype)rJemptyfill_cuda)sizerar device_idrPrPrQ _build_tensors r cs2|dkr |}tjfddt|D|d|S)Ncsg|]}qSrPrPr{_dim_sizerPrQrsz*_build_multidim_tensor..)r r)rJrranger )rrrarrPrrQ_build_multidim_tensorsrcCstjjjddSNT) record_shapes)rJautogradrrrPrPrPrQ_create_autograd_profilersrcCstjjtjjjgddSNT) activitiesr)rJrrProfilerActivityCPUrPrPrPrQ_create_torch_profilers rc@s*eZdZdZeddZedddZdS) BarrierrcCsBd|_tjtjdd}t|D]}ttj||q$dS)Nrrbarrier) barrier_idrrrrlistdirunlink)cls barrier_dirf_namerPrPrQinitsz Barrier.initNr<c Cs&|dkrt}|jd7_tjtjdd}tt}tj||}t ,t |d}| t|jW5QRXW5QRXt }d}t Tt |D]B} t tj|| d$}|} t| |jkr|d7}W5QRXqW5QRX||krq"t ||krtdt dqdS) Nr?rrrrrzbarrier timeout皙?)rrrrrrrrgetpidrrwritetimer readr RuntimeErrorsleep) r"wait_fortimeoutr#pidZ barrier_filef start_timeZarrivedr$datarPrPrQsyncs,  $z Barrier.sync)Nr<)rWrXrYr classmethodr%r4rPrPrPrQrs  rcs\eZdZefddZfddZfddZeddZed d Z ed d Z Z S) TestDistBackendcstttjd<tdS)N MASTER_ADDR)rr7rrrC setUpClass)r"rNrPrQr8 szTestDistBackend.setUpClasscs"tttg|_dSr\)rCsetUprrr%Zskip_return_code_checksrLrNrPrQr9s zTestDistBackend.setUpcsttdSr\)r rCtearDownrLrNrPrQr:szTestDistBackend.tearDowncCsdjt|jdS)Nz {}{file_name}) file_name)formatr2r;rLrPrPrQ init_methodszTestDistBackend.init_methodc Cs.ddlm}|dtdkr6tjs6ttdj ||}||_ ||_ tjrtj t |jkrttd|jj z8t|t}t|d}tj|jtt |j|j |dWnBtk r} z$d | jdkrttd j W5d} ~ XYnX|||||ttddS) Nr)_set_ddp_with_replicated_tensorTncclZno_cudaz multi-gpu-secondsr=rrrr/Z recompileZbackend_unavailable).torch.nn.parallel._replicated_tensor_ddp_utilsr>rsrJr Z is_availablerexitrZ exit_coderr; device_countrrCUSTOM_PG_TIMEOUTgetdefault_pg_timeoutrrinit_process_groupr=r,args_barrierZrun_testdestroy_process_group) r"rrr;piper>rMZpg_timeout_secondsr/erPrPrQ_run#s<     zTestDistBackend._runcCs tjdS)NrrrrLrPrPrQrOszTestDistBackend.world_size) rWrXrYr5r8r9r:propertyr=rOrrZrPrPrNrQr6 s   +r6c@seZdZGdddZdS)DistributedTestc@sh1eZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZeedkdeed d d!d"Zeeedkdd#d$Zeedkdd%d&Zd'd(Zeejd)eejd)ed*ed+d,d-Z eejd)eejd)ed*d.d/Z!eeejd0kd1ed2ed3ed+d4d5Z"eeejd0kd1ed2e#d6d7Z$eeejd0kd1ed2ed3ed3d8d9Z%eeejd0kd1ed2ed3ed3d:d;Z&eeejd0kd1ed2ed3ed3dd?Z(eeejd0kd1ed2ed3ed3d@dAZ)eeejd0kd1ed2ed+dBdCZ*eeejd0kd1ed2ed+dDdEZ+ed+dFdGZ,eeejd0kd1ed2ed+dHdIZ-eeejd0kd1ed2ed3ed3dJdKZ.e#eedLkdMe/dNdOdPdQZ0e#eedLkdMe/dNdOdRdSZ1e#eedLkdMe/dNdOdTdUZ2e#eeedLkdMe/dNdOdVdWZ3eedkdXdYdZZ4eedkdXd[d\Z5eedLkdMe/dNdOd]d^Z6eedLkdMe/dNdOd_d`Z7eedLkdMe/dNdOdadbZ8eedLkdMe/dNdOdcddZ9e#eedLkdee/dNdOddgdhZ:e#eedLkdee/dNdOdidjZ;e#eedLkdee/dNdOdkdlZp,e?dndodpZ@dqdrZAeedLkdsdtduZBeedLkdvdwdxZCeedLkdvee=dmee>pe?dndydzZDd{d|ZEeeejFd}ked~ddZGeeejFd}ked~ddZHeeejFd}ked~ee=dee>pe?dnddZIddZJeedLkdvddZKeedLkdvddZLeedLkdvee=dee>pe?dnddZMddZNeedLkdddZOeedLkdddZPeedLkdee=dee>pe?dnddZQeedLkdddZRdddZSeedLkdddZTeedko@edLkde#ddZUeeedLkdddZVeedLkdddZWeedLkde#ddZXdddZYeedLkdeeejFdkedddZZeedLkdeeejFdkede#ddZ[eedLkdeeejFdkedddZ\eedLkdeeejFdkedddZ]eedLkdeeejFdkedddZ^eedLkdeeejFdkededdZ_eedLkdeeejFdkededdZ`eedLkdeeejFdkededdZaeedLkdeeejFdkededdZbeedLkdeeejFdkeddd„ZceedLkdeeejFdkedddĄZdeedLkdeeejFdkedddƄZeeedLkdeeejFdkedddȄZfdddʄZgeedLkdeeejFdkeddd̄ZheedLkdeeejFdkede#dd΄ZieedLkdσeeejFdkede#ddфZjdddԄZkeedLkdՃe#ddׄZle#eejd)ddلZmddfddfdڜdd܄ZnddfeojpdfddބZqeedLkdddZreedLkdddZseedk ovedLkde#ddZteedk oedLkde#ddZueedLkdddZveedLkdddZweedk oedLkde#ddZxeedLkdddZyeedLkdddZzeedLkdddZ{eeedLkdddZ|eeedLkdddZ}eeedLkdddZ~eeedLkdddZeedLkdddZeedLkdddZeedLkdddZeedLkdddZddZeedkdddZeedkde#d d Zed d Zed dZeddZeddZeedLkdddZdddZedhddZedhddZedhddZedhddZeedhdd Zeedhd!d"Zeedhd#d$Zeedhd%d&Zedhd'd(Zedhd)d*Zedhd+d,Zedhd-d.Zddfeojpfd/d0ZeedLkdeed1kd2d3d4ZeedLkdeed1kd2d5d6ZeedLkd7e#d8d9ZeedLkdeed1kd2d:d;ZeedLkd7e#d<d=ZeedLkdeed1kd2ed>d?ZeedLkdeed1kd2d@dAZddBdCZeedLkdeed1kd2dDdEZeedLkdeed1kd2dFdGZeedLkd7e#dHdIZeedLkdeed1kd2edJdKZeedLkdeed1kd2dLdMZddfeojpfdNdOZeedLkddPdQZeedLkdRe#dSdTZeedLkddUdVZeedLkdRe#dWdXZeeedLkddYdZZeedLkdd[d\ZeedLkd]e#d^d_Zdd`daZeedLkdbe#dcddZeedLkdbe#dedfZdgdhZeojpfdidjZeeejFdkkedldmdnZeeejFdkkedldodpZeeeejFdkkedldqdrZeeejFdkkedldsdtZeeejFdkkedldudvZddfeojpfdwdxZddfeojpfdydzZddfeojpfd{d|Zeed}kd~ddZeedLkde#ddZeed}kd~ddZeedLkde#ddZeed}kd~ddZeedLkde#ddZeed}kd~ddZeedLkde#ddZeed}kdddZeedLkdeƐddZeed}kdddZeedLkdeƐddZeed}kd~eddZeedLkde#eddZeed}kd~eddZeedLkde#eddZeed}kdeddZeedLkdeeƐddZeed}kd~ddZeedLkde#ddZeed}kd~ddZeedLkde#ddZeed}kdddZeedLkdeƐddZՐdddZe#eed}kdeed1kdddZee#eed}kdddZee#eed}kdddZeeejFdkedddZeeeejFdkedddZeeejFdkedddÄZܐdĐdńZeed}kdƃeedLkdǃe#dȐdɄZeojpfdʐd˄Zeed}kdƃeedLkd̃eed1kd̓e#dΐdτZeed}kdƃeedLkd̃eed1kd̓e#dАdфZdҐdӄZeedLkdԃe#dՐdքZeojpfdאd؄ZeedLkdكe#dڐdۄZeedLkdكe#dܐd݄Zdސd߄ZddZddZd ddZddZd!ddZd"ddZd#ddZeedLkdddZeedLkdddZeeejdkd1edddZeedLkdddZeeejdkd1edeeejdddZeeejdkd1edeeejdddZddZeedLkpJed1kded+edddgedddged ddgd d ZeedLkped1kded+ed ddgd d ZeedLkped1kded+ed ddgddZd$ddZeeejdkd1edeeejdddZeeejdkd1edeeejdddZeeejdkd1edeeejdddZeeejdkd1edeedeeejdddZd%ddZd&ddZd'd d!Zeed}koedLkoedkd"eed+d#d$Zeed}koedLkoedkd"eed+d%d&Zeed}ko edLko edkd"eed+d'd(Z eed}koLedLkoLedkd"eed+d)d*Z eed}koedLkoedkd"eed+d+d,Z eeejdkd1ede#d-d.Z d(d/d0Z eeejdkd1ede#d1d2Zd)d3d4Zd5d6Zd7d8Zd9d:Zd;d<Zd=d>Zed+eeejdkd1edd?d@Zed+eeejdkd1eddAdBZdCdDZed3eeeejdkd1eddEdFZed3eeeejdkd1eddGdHZed+eeejdkd1eddIdJZeeejdkd1ede#dKdLZeeejdkd1ede#dMdNZeeejdkd1ede#dOdPZeeejdkd1ede#dQdRZeeejdkd1ede#ed+dSdTZ eeejdkd1ede#dUdVZ!eeejdkd1ede#dWdXZ"dYdZZ#eedLkdd[d\Z$eeejdkd1ede#d]d^Z%eedLkdd_d`Z&e'dadbZ(e)j*dffdcddZ+edLhedLhed+dedfZ,edLhedLhed+dgdhZ-edLhedLheeejddidjZ.edLhedLhed+dkdlZ/eeejdkd1edeeejddmdnZ0d*dodpZ1eejd)e2eejdejdqe3drdsdtgdudvdwZ4eejd)e2eejdejdqe3dtdrdsgdudxdyZ5d+dzd{Z6eejd)e3dtdrdsgdud|d}Z7eejd)e3dtdrdsgdud~dZ8ddZ9ed+eeejdkd1edddZ:ed+eeejdkd1edddZ;ddZ$pe?dnddZ>ed+eeejdkd1edddZ?ddZ@ed+eeejdkd1edddZAed+eeejdkd1edddZBed+eeejdkd1edddZCed+eeejdkd1edddZDd,ddZEeejd)e2eejdejdqe3dtgduddZFeejd)e2eejdejdqe3dtgduddZGd-ddZHeejd)eejd)ed+ddZIe3drdsdtgdueejd)eejd)ed+ddZJeejd)eejd)ed+ddZKeejd)eejd)ed+ddZLeejd)eejd)ed+ddZMe3drdsdtgdueejd)eejd)ed+ddZNeejd)eejd)ed+ddZOe3drdsdtgdueejd)eejd)ed+ddZPedhddZQddZRddZSeejd)eejd)ed+ddZTeejd)eejd)ed+ddZUd.ddZVddZWeejd)eejd)ed+ddZXeejd)eejd)ed+ddZYddÄZZeejd)eejd)ed+dĐdńZ[eejd)eejd)ed+dƐdDŽZ\dȐdɄZ]eeejdkd1eded+dʐd˄Z^eeejdkd1eded+d̐d̈́Z_eeejdkd1eded+dΐdτZ`d/dАdфZaeejd)eejd)ed+dҐdӄZbedhedhee>+pBe?dԃdՐdքZcedhedhdאd؄ZddِdڄZeefeejd)eejd)eeejddېd܄Zgefeejd)eejd)eeejddݐdބZhedhedhdߐdZiedhedheee>,pRe?dԃddZjedhedheddZkeejd)eejd)e3dsgdued+ddZleeejdkd1ede3dsgdued+ddZmddZne3drdsdtgdueejd)eejd)ed+ddZoe3drdsdtgdueejd)eejd)ed+ddZpeeejdkd1eded+ddZqeeejdkd1eded+ddZred+eeejdkd1edddZsed+eeejdkd1edddZted+eeejdkd1edddZued+eeejdkd1edddZvddZwed+eeejdkd1edddZxed+eeejdkd1edddZyddZzed+eeejdkd1edddZ{ed+eeejdkd1edddZ|ed+eeejdkd1edddZ}ed+eeejdkd1edd d Z~ed+eedLk0oedkd d d Zed+eeejdkd1edddZeejd)eejd)ed+ddZddZeeejdkd1edeed1kdeeejdddZdfS(0zDistributedTest._DistTestBasecOstj||dSr\)rr4)rMrJkwargsrPrPrQrKVsz&DistributedTest._DistTestBase._barriercKs:ddg}tj|f|}t}||kr0gd|fS|||fS)Nr?rA)r new_grouprrMrSgroupgroup_idrrPrPrQ_init_group_testYs  z.DistributedTest._DistTestBase._init_group_testcKs0ttdt}tjf|}t}|||fSr)listrrrrTrrUrPrPrQ_init_full_group_testbs z3DistributedTest._DistTestBase._init_full_group_testcCs,ttdt}tjj}t}|||fSr)rYrrrrVWORLDrrMrVrWrrPrPrQ_init_global_testhsz/DistributedTest._DistTestBase._init_global_testc sdd|jD}|jD]\}}||||qt|}t|}t||D]\fddttD}t |fddttD} |D]} || qt | | D]} || qqZdS)NcSsi|]\}}||qSrPrP)r{kvrPrPrQ pszGDistributedTest._DistTestBase._verify_buffers_equal..csg|]}tqSrPrJ empty_liker)buf1rPrQrxszGDistributedTest._DistTestBase._verify_buffers_equal..csg|]}tqSrPrar)buf2rPrQr|s) r named_buffers assertEqualrYbuffersziprrr all_gather) rMm1m2Z m1_buf_dictrxbufZ m1_buffersZ m2_buffersZ gathered_bufsZgathered_bufs_m2rHrP)rcrdrQ_verify_buffers_equalns$        z3DistributedTest._DistTestBase._verify_buffers_equalc Cst\}}t|}W5QRXdd}dddddg}|D]}||}|||qBdd d g}|D]}||}|||qjdS) NcSs d||tjkrtj|ndfS)Nz env:%s=%sN/ArPvarrPrPrQ format_lineszRDistributedTest._DistTestBase.test_dump_DDP_relevant_env_vars..format_liner7 MASTER_PORTrZNCCL_TOPO_DUMP_FILEZNCCL_ASYNC_ERROR_HANDLINGZxxxZyyyZzzz)r*rgetvalue splitlinesZassertIn assertNotIn)rMoutrlinesrqvarsrplinerPrPrQtest_dump_DDP_relevant_env_varss( z=DistributedTest._DistTestBase.test_dump_DDP_relevant_env_varsc Cstjtjdd}tt}t}ttj||d}| tt W5QRX| t }t |D]6}ttj||d}|t|W5QRXqr|t||| t dkrt |D]}ttj||q| dS)Nrtest_dirrr&r)rrrrrr(rrrr)rrKsetr addrr+rflenr!)rMr{r0 num_processesr1Z all_ranksr$rPrPrQ test_get_ranks   z+DistributedTest._DistTestBase.test_get_rankc Cstdkrddg}nddg}t|}t}|t|t|kr`|t||n"|t dt|W5QRXdS)NrAr?rzInvalid process group specified) rrrTrslowerrf get_backendrassertRaisesRegexr,)rMrVrWZ backend_strrPrPrQtest_get_backends    z.DistributedTest._DistTestBase.test_get_backendc Cst}|tt||tt||tdtdW5QRX|tdtdW5QRX|ttdW5QRX|ttdW5QRX|ttdgW5QRXdS)NzInvalid backend: 'undefined'Z undefinedzInvalid backend: 'xYz'ZxYzrhgloo) rsrrfrrupperr ValueError assertRaises)rMrrPrPrQtest_Backend_enum_classs   z5DistributedTest._DistTestBase.test_Backend_enum_classcCs>tdkrddg}nddg}t|}|t|dS)NrAr?r)rrrTrKrLrMrVrWrPrPrQtest_destroy_groups    z0DistributedTest._DistTestBase.test_destroy_groupcCstdkrddg}nddg}t|}t|krd|t|d|t|ttdkn$|t|d|t|ddS)NrAr?rr)rrrTrrf assertTruerYrrrPrPrQtest_get_rank_size_groups    z6DistributedTest._DistTestBase.test_get_rank_size_groupcCs$|\}}}|t|dSr\)rZrKrrLrMrrWrPrPrQtest_destroy_full_groupsz5DistributedTest._DistTestBase.test_destroy_full_groupcCs>|\}}}|t|t|t|tdSr\)rZrfrrrrrPrPrQtest_get_rank_size_full_groupsz;DistributedTest._DistTestBase.test_get_rank_size_full_groupc Cst|}|dkr|t|}ttjjkr@|td}n |td}|t |W5QRX|j t|ddndS)Nrzfailed to pass monitoredBarrierz (Timed out|closed|timeout) r')delta) rrr* total_secondsget_debug_level DebugLevelDETAILr ExceptionrassertGreaterAlmostEqual)rMrWr/Z local_rank expected_time exception_ctxrPrPrQ_test_barrier_timeouts  z3DistributedTest._DistTestBase._test_barrier_timeoutrz#Only gloo backend supports timeoutsfile://zRequires file:// initialization method. Both tcp:// and env:// rely on the TCP store for which reinitialization has proven racy.cCs\t|jttjddtdd}tjtt ttjd|j |d| tj j |dS)Nr)r.r?r@rB)rrLrKrrrrrIrtrsrrrVr[)rMr/rPrPrQtest_barrier_timeout_globals  z9DistributedTest._DistTestBase.test_barrier_timeout_globalcCs4tdd}|j|d\}}}|dk r0|||dS)Nrnr@r/)rrXrrMr/rrWrPrPrQtest_barrier_timeout_group1s z8DistributedTest._DistTestBase.test_barrier_timeout_groupcCs4tdd}|j|d\}}}|dk r0|||dS)Nr?r@r)rrZrrrPrPrQtest_barrier_timeout_full_group9s z=DistributedTest._DistTestBase.test_barrier_timeout_full_groupcCstdkrd}n tdkrd}nttjdkr.d}||d\}}}|dkrJdS|dkrd|t|tj|dkr~|t|tj|||t ||t |t |t |}t j |td|d }tj||d|d|tddd|d dS) Nrr?rrrArarsrcrVcpu)rsr.rrr^rZProcessGroupGlooProcessGroupNCCLrfrr~rrJr  set_devicer  broadcastto)rMZ initializerZ new_backendrVrWrZ group_rankrqrPrPrQ_test_group_override_backendCs(  z:DistributedTest._DistTestBase._test_group_override_backendgpurhrAcCs||jdSr\)rrXrLrPrPrQtest_backend_group`sz0DistributedTest._DistTestBase.test_backend_groupcCs||jdSr\)rrZrLrPrPrQtest_backend_full_groupgsz5DistributedTest._DistTestBase.test_backend_full_groupsubgroupzThe z< backend does not support creating subgroups on CUDA devicesricCsfd}t|\}}t}||||t||||t||D]}t|qRdSr) r new_subgroupsrrfr r~ assertFalse_rank_not_in_grouprL)rMZ subgroup_size cur_subgroup subgroupsrrrPrPrQtest_new_subgroupsmsz0DistributedTest._DistTestBase.test_new_subgroupsc Cs&|tdtdW5QRXdS)Nz3The arg 'group_size' must not exceed the world sizedrrrrrLrPrPrQ0test_new_subgroups_group_size_exceeds_world_sizes zNDistributedTest._DistTestBase.test_new_subgroups_group_size_exceeds_world_sizec Cs&|tdtdW5QRXdS)Nz0The world size must be divisible by 'group_size'rhrrLrPrPrQ9test_new_subgroups_world_size_not_divisible_by_group_sizes zWDistributedTest._DistTestBase.test_new_subgroups_world_size_not_divisible_by_group_sizec Cs|\}}}ttt}||d}tjddgddggd\}}|dkrX||nR||d|t |d|dks|dkr|||dn|||d|D]}t |qdS)NrrAr?rhZranks_per_subgroup_listri) r]rrrrsnew_subgroups_by_enumeration assertIsNonerfr r~rL) rMrVrWr rank_to_GPUr rrrrPrPrQ!test_new_subgroups_by_enumerations   z?DistributedTest._DistTestBase.test_new_subgroups_by_enumerationc Csd|\}}}ttt}||d}t|}|tdtjddg|dggdW5QRXdS)NrRThe new group's rank should be within the the world_size set by init_process_groupr?rAr)r]rrrrsrr,r)rMrVrWrrr rrPrPrQ?test_new_subgroups_by_enumeration_input_rank_exceeds_world_sizes z]DistributedTest._DistTestBase.test_new_subgroups_by_enumeration_input_rank_exceeds_world_sizec CsB|\}}}|tdtjddgddggdW5QRXdS)Nrrr)r]rr,rrr\rPrPrQ5test_new_subgroups_by_enumeration_negative_input_rankszSDistributedTest._DistTestBase.test_new_subgroups_by_enumeration_negative_input_rankc Cs8|td"tjdgddgddggdW5QRXdS)Nz$Rank 1 has appeared in both subgrouprr?rArhr)rrrrrLrPrPrQ&test_new_subgroups_overlap_not_allowedszDDistributedTest._DistTestBase.test_new_subgroups_overlap_not_allowedc CsNt}ttt}||d}ttjdddddttj dddd |}| D]}t |j|_q\tj| dd| D]}||jt |jq| D]}t |j||_qtjddgd d }tj| |dt|s | D]}||jt |jd qn*| D] }||jt |j|q(dS) Nrrhr?)Z kernel_sizepaddingrnFr=)params process_groupr?ranksr?)rrrrrsrE SequentialConv2drrFr  parametersrJ ones_liker3model_averaging_utilsaverage_parametersrfrTr)rMrrr rr group_ncclrPrPrQtest_average_parameterss<       z5DistributedTest._DistTestBase.test_average_parameterscCst}t}t|t}||d}tjdddd|}t| }t |j |}t |j t t||}d} dD]} tj| | d} tdd D]p} t||_ | D]} t |j | _q| | | | kr| | | dkr||j |q||j |qq|dS) Nrr?rnFr=ri period warmup_stepsr)rrrrrsrErFr nextrrJrr3rr averagersPeriodicModelAveragercopydeepcopygradrrfrMrrrr rparamrqZexpected_avg_tensorrraveragersteprrPrPrQtest_periodic_model_averager s(     z:DistributedTest._DistTestBase.test_periodic_model_averagerc Cst}t}t|t}||d}tjdddd|}t| }t j j | dd}d}d D]&} t j|| d } tdd D]} |jD]6} | d D](} t |j|| _t |j|| _qq| |j| | krB| | |dkrB|jD]J} | d D]<} | jdkrq||jt |jtt||qqq|jD]@} | d D]0} | jdkrhqT||jt |j|qTqHqqddS) Nrr?rnFr=r'lrrirrrr)rrrrrsrErFr rrrJoptimSGDrrrZ param_groupsrr3rrrfr)rMrrrr rroptrrrrZ param_grouprrPrPrQ(test_periodic_model_averager_param_group,s6          .   zFDistributedTest._DistTestBase.test_periodic_model_averager_param_groupcCs,t}t}t|t}||d}tjdddd|}t| }t |j |}t |j t t||}d} dD]} tjt| |fg| d} tj| | d } tdd D]t} t||_ | D]} t |j | _q| | | | kr| | | dkr||j |q||j |qq|dS) Nrr?rnFr=rirperiod_group_size_dictrrr)rrrrrsrErFr rrrJrr3rrhierarchicalSGDHierarchicalModelAveragerrrrrrrrrfrrPrPrQNtest_1_level_hierarchical_model_averager_equivalent_to_periodic_model_averagerPs0      zlDistributedTest._DistTestBase.test_1_level_hierarchical_model_averager_equivalent_to_periodic_model_averagercCs\ddlm}t}t}t|t}||d}tjdddd |}t | }t |j|}d} d} d} d } d } d }t| | f| | f||fg}tj|| d }|j| }|j| }t||}t||}|| | ttt| }|| | ttt| }||||||t |jt|| }t |jt|| }t |jtt||}tdd D]}t||_| D]}t |j|_q|| |d ks|dkr||j|nj|dks|dkr||j|nF|dks6|dks6|dks6|dkrF||j|n||j|qdS)Nr)_pg_group_ranksr?rnFr=r<rArirrrr)"torch.distributed.distributed_c10drrrrrrsrErFr rrrJrr3rrrZperiod_process_group_dictrYkeysnparrayrtolistrfrrrrr)rMrrrrr rrrqrZsubgroup_size1Zsubgroup_avg_period1Zsubgroup_size2Zsubgroup_avg_period2Zglobal_avg_periodrrZ subgroup1Z subgroup2Zreal_group_ranks_res1Zreal_group_ranks_res2Zexpect_group_ranks_res1Zexpect_group_ranks_res2Z$expected_avg_tensor_within_subgroup1Z$expected_avg_tensor_within_subgroup2Zexpected_global_avg_tensorrrrPrPrQ(test_3_level_hierarchical_model_averagerwsd      ""    (zFDistributedTest._DistTestBase.test_3_level_hierarchical_model_averagerr?zNCCL Batch Send Recv Only)rArzNeed NCCL 2.7+ for send/recvcCs\|t}t}t|t}||d}tj|g}ddt |D}ddt |D}dD]}|t j d<t d|D]} t |d|d | } t | dd |d  d || <t | dd |d  ||| <ttj|| | } || ttj| | } || q~t|} | D]}|qt d|D]} ||| || q2qf|dS) NrcSsg|]}dqSr\rPrrPrPrQrszMDistributedTest._DistTestBase.test_batch_isend_irecv_nccl..cSsg|]}dqSr\rPrrPrPrQrs)10NCCL_BLOCKING_WAITr?r rrar )rKrrrrrsrJr rrrrr r P2POpirecvappendisendbatch_isend_irecvwaitrf)rMrrrr  p2p_op_listZ recv_tensorsexpected_tensorsvalr send_tensorrecv_opsend_opreqsreqrPrPrQtest_batch_isend_irecv_nccls2        z9DistributedTest._DistTestBase.test_batch_isend_irecv_ncclc Cs|t}t}t|t}||d}tj|g}t ||d}t |d|d}t tj ||d|}t tj ||d||} t || g} | D] } | q|dS)Nrrrrr?)rKrrrrrsrJr rr rrrrr) rMrrrr r r  recv_tensorrr rrrPrPrQ)test_batch_isend_irecv_ring_exchange_nccls     zGDistributedTest._DistTestBase.test_batch_isend_irecv_ring_exchange_ncclc Cs|tt}ttt}||d}g}|dkrt|d|d}t|dd|d}ttj |d}| |ttj |d}| |t |} | D] } | q|dS)Nrr?rrr)rKrrrrrrsr rrrrrr) rMrrr r r rr rrrrPrPrQ test_batch_isend_irecv_self_nccls"     z>DistributedTest._DistTestBase.test_batch_isend_irecv_self_ncclc Cs|tt}ttt}||d}tj |g}|dkrPd}n |dkr\d}|dkrt |d|d}t |dd|d}t tj ||}| |t tj||} | | t|} | D] } | q|dS)Nrr?rA)r?rArrr)rKrrrrrrsrJr rr rrrrrr) rMrrr r Zpeerr rr rrrrPrPrQ(test_batch_isend_irecv_no_rank_zero_nccls,      zFDistributedTest._DistTestBase.test_batch_isend_irecv_no_rank_zero_ncclzGLOO Batch Send Recv CPUc Cs|t}g}tdtD]^}||kr0q"t|d}t|ddd}ttj||}||ttj ||}||q"t |}|D] } | q|dS)Nrr?rr rKrrrrr rrrrrr rMrr rr rr rrrrPrPrQtest_batch_isend_irecv_gloo8s      z9DistributedTest._DistTestBase.test_batch_isend_irecv_glooc Cs|t}g}tdtD]f}||kr0q"t|d}t|ddd}tjtj|||d}||tjtj |||d}||q"t |}|D] } | q|dS)Nrr?rrtagrrrPrPrQ test_batch_isend_irecv_gloo_tagsOs      z>DistributedTest._DistTestBase.test_batch_isend_irecv_gloo_tagsc Cs\|t}|dkrX|td.t|d}ttj|d}t|gW5QRXdS)NrzTensors must be CUDA and denser?) rKrrrr,r rrr)rMrr rrPrPrQ!test_batch_isend_irecv_tensor_errfs z?DistributedTest._DistTestBase.test_batch_isend_irecv_tensor_errc Csz|t}|dkrvttt}||d}|td2t|d|d}t tj |d}t |gW5QRXdS)Nrz^Invalid ``op``r?r) rKrrrrrsrr,r rrr)rMrrr r rrPrPrQtest_batch_isend_irecv_op_errts z;DistributedTest._DistTestBase.test_batch_isend_irecv_op_errc CsB|t}|dkr>|tdtddgW5QRXdS)Nrz^Invalid ``p2p_op_list``r?rA)rKrrrr,rrMrrPrPrQ"test_batch_isend_irecv_op_list_errs z@DistributedTest._DistTestBase.test_batch_isend_irecv_op_list_errc Cs|t}ttt}||d}tjddgdd}tjddgdd}|dkr|tdDt |d}t tj |d|}t tj |d|}t ||gW5QRXdS)Nrr?rrr?z"All ops need to use the same group) rKrrrrrsrTrr,r rrr) rMrrr  group_gloorr Z send_op_glooZ send_op_ncclrPrPrQ(test_batch_isend_irecv_mixed_backend_errs  zFDistributedTest._DistTestBase.test_batch_isend_irecv_mixed_backend_errzNCCL Send Recv OnlyNc Csdt}t}t|t}||d}tj|t|d|d}|dk rN|nt }|}t d|D]j} | |krt d|D]} | |krqzt || qzqdt| d} t| dd|d} t | | | | | qd|W5QRX|dk r`t} | tkr`| d| dfD]L}t||}||dd t tD}|D]}||j|kqDqdS) Nrr?rrr:send:recvcSsg|]}|dgdgqSr?rhrPr{rrPrPrQrszFDistributedTest._DistTestBase._test_send_recv_nccl..)rrrrrsrJr rr rrsendrecvrfrKr&SEND_RECV_PROFILING_SUPPORTED_BACKENDSrr input_shapes)rM profiler_ctxrrrr rqZ profiler_clsprofrdstexpected_tensor output_tensorr event_namerexpected_shapesr|rPrPrQ_test_send_recv_ncclsD          z2DistributedTest._DistTestBase._test_send_recv_ncclcCs |dSr\)r1rLrPrPrQtest_send_recv_ncclsz1DistributedTest._DistTestBase.test_send_recv_ncclcCstjjjdd}||dSr)rJrrrr1rMr*rPrPrQ%test_send_recv_nccl_autograd_profilerszCDistributedTest._DistTestBase.test_send_recv_nccl_autograd_profilerzKineto in fbcode causes hangzYtorch.profiler not enabled for mac/windows: https://github.com/pytorch/pytorch/pull/56124cCs.tjjtjjjtjjjgdd}||dSr)rJrrrrCUDAr1r3rPrPrQ"test_send_recv_nccl_torch_profilers z@DistributedTest._DistTestBase.test_send_recv_nccl_torch_profilerc Csbt}|d}t|}|dk r$|nt}|}tdtD]l}||krvtdtD]}||krfqXt||qXq>|d} t| } t| dd} t| ||| | q>W5QRX|dk r^t } | t kr^| d| dfD]v} t | |}t dd|D}td}|||d d ttD}|D]"}| |j| |j|kq8qdS) Nr?rrrr"r#css|] }|jVqdSr\countr{rNrPrPrQr sz@DistributedTest._DistTestBase._test_send_recv..cSsg|]}|dgdgqSr$rPr%rPrPrQrszADistributedTest._DistTestBase._test_send_recv..)rrr rrrr&r'rfrr(rrris_asyncr))rMr*rZ send_sizerqctxr+rr,Z recv_sizer-r.rr/r event_countexpected_event_countr0r|rPrPrQ_test_send_recvs<         z-DistributedTest._DistTestBase._test_send_recvz,Nccl send/recv tested by test_send_recv_ncclcCs|jdddSNr*)r>rLrPrPrQtest_send_recvsz,DistributedTest._DistTestBase.test_send_recvz,NCCL send/recv tested by test_send_recv_ncclcCst}|j|ddSr?)rr>rMautograd_profiler_ctxrPrPrQ test_send_recv_autograd_profilersz>DistributedTest._DistTestBase.test_send_recv_autograd_profilercCst}|j|dSr?)rr>rMtorch_profiler_ctxrPrPrQtest_send_recv_torch_profiler"s z;DistributedTest._DistTestBase.test_send_recv_torch_profilerc sjt}d}t||d}t}t}|dk r0|nt}|}tdtD]} | |krtdtD]} | |krrqddD]n} t|dd} | dkrt| } || n,| dkrt | } | | } || | | | qvqdqJt|| t|| qJW5QRX|dk rft}|tkr|d|d fD]d}t||}|td d |Dd td |D](}| |j||j|gdgqvq>tt|t|fdfddttD}t|g}|D]}||7}qddlm}|dd||D}|tt||d td gt|| dS)Nr<rr)r'rrr'rr"z:recvAnySourcecss|] }|jVqdSr\r7rzrPrPrQr[szKDistributedTest._DistTestBase._test_send_recv_any_source..rAr?rhcsg|]}tqSrPrarZrecv_ranks_tensorrPrQrgszLDistributedTest._DistTestBase._test_send_recv_any_source..)groupbycSsg|]\}}tt|qSrP)r~rY)r{keyrVrPrPrQrss)!rrr rYrrrr'rrrZ _source_rankrrcrr&rr(rrfrr:r)rJcatrqrir itertoolsrIsortr~rK)rMr*rsend_recv_sizerqZ recv_ranksZ irecv_ranksr;r+r,r'r.Zsenderworkrr/rr|Zglobal_recv_ranksZglobal_recv_ranks_listrIZ frequencyrPrHrQ_test_send_recv_any_source/sv               z8DistributedTest._DistTestBase._test_send_recv_any_sourcezsendrecv anysourcez+ does not support send/recv from any sourcecCs|jdddSr?)rPrLrPrPrQtest_send_recv_any_source|sz7DistributedTest._DistTestBase.test_send_recv_any_sourcecCst}|j|ddSr?)rrPrBrPrPrQ+test_send_recv_any_source_autograd_profilerszIDistributedTest._DistTestBase.test_send_recv_any_source_autograd_profilerz!Kineto in fbcode code causes hangcCst}|j|dSr?)rrPrErPrPrQ(test_send_recv_any_source_torch_profilers zFDistributedTest._DistTestBase.test_send_recv_any_source_torch_profilerc Cs`t}t}d}t||d}|dk r,|nt}|z}td|D]h}||krtd|D]>} | |krfqXt|dd} tj| | | d|| |  qXqBtj |||dqBW5QRX|dk r\t } | t kr\| d| dfD]t} t | |} tdd | D}td }|||| D]6}||j||j| ||j|gd gq"qdS) Nr<rrrrr"r#css|] }|jVqdSr\r7r9rPrPrQrszIDistributedTest._DistTestBase._test_send_recv_with_tag..r?rh)rrrr rrr'rrcrr&rr(rrrfr:rxr))rMr*rrrNrqr;r+r,rr.rr/rr<r=r|rPrPrQ_test_send_recv_with_tags6        z6DistributedTest._DistTestBase._test_send_recv_with_tagcCs|jdddSr?)rTrLrPrPrQtest_send_recv_with_tagsz5DistributedTest._DistTestBase.test_send_recv_with_tagcCst}|j|dSr?)rrTrBrPrPrQ)test_send_recv_with_tag_autograd_profilerszGDistributedTest._DistTestBase.test_send_recv_with_tag_autograd_profilercCst}|j|dSr?)rrTrErPrPrQ&test_send_recv_with_tag_torch_profilers zDDistributedTest._DistTestBase.test_send_recv_with_tag_torch_profilerc Cst}t}|dk r|nt}|t}|dkrfddtd|D}|D]}|||qHn(t|d}t |d| |t|d| W5QRX|dk rt } | t kr|dkr| dn| d} t| |} td d | D} |dkr tdnd} | | | d d tdtD}| D]R}||j| |j| |dkrt||j|kn| |j||q6dS) NrcSsg|]}tt|d|qSr<)rrr )r{destrPrPrQrsz=DistributedTest._DistTestBase._test_isend..r?rr<r"r#css|] }|jVqdSr\r7r9rPrPrQrsz.cSsi|]}||gdgqSrhrPr{r&rPrPrQr`sz=DistributedTest._DistTestBase._test_isend..)rrrrrrr is_completedr r'rfrKrr(rrr:rxr)values)rMr*rrr;r+requestsrequestrqrZexpected_event_namerr<Zexpected_countr0r|rPrPrQ _test_isendsF          z)DistributedTest._DistTestBase._test_isendzNccl does not support isendcCs|jdddSr?)r`rLrPrPrQ test_isendsz(DistributedTest._DistTestBase.test_isendcCst}|j|ddSr?)rr`rBrPrPrQtest_isend_autograd_profilersz:DistributedTest._DistTestBase.test_isend_autograd_profilercCst}|j|ddSr?)rr`rErPrPrQtest_isend_torch_profilersz7DistributedTest._DistTestBase.test_isend_torch_profilerzNccl does not support irecvcst}t}|dkrddtd|Dfddtd|D}td|D]D}||d|||d||dt|dqNnt|d}t |d| dS)NrcSsg|]}t|dqS)rr r{rrPrPrQrsz.r?cs g|]}t|d|qSr?)rrrer rPrQrsr<) rrrrrrr\rfr r&rK)rMrrr^rrqrPrgrQ test_irecvs    z(DistributedTest._DistTestBase.test_irecvFc Cstjddftjddftjddftjddftjddftjddftjd dffD]6\}}} | r`|s`qJ|D]} t| d ||} |r| ||d } || kr|rt } d | _ | | _ |d d|j| g| n|d dt j| | |qdt| d d |} |r| ||d } |r:t } d | _ | | _ |d d|j| g| n|d dt j| | ||| | || | tdqdqJ|dS)Ng|۽Fg0.+gTrgjg4&k r?r :broadcastr)rJfloatdoubleZhalfZint8Zuint8rrr r rZBroadcastOptionsZ rootTensorZrootRank call_dist_oprrfr nemaxrqrK)rMrVrWrr rZ with_optionsrraZ requires_cudarr-optsrqrPrPrQ_test_broadcast_helper,s|           z4DistributedTest._DistTestBase._test_broadcast_helperz!Nccl does not support CPU tensorscCs |\}}}||||dSr\)r]rqr\rPrPrQtest_broadcasttsz,DistributedTest._DistTestBase.test_broadcastz2Only Gloo and Nccl backend supports CUDA allReducecCsJ|\}}}ttt}||d}tj|||||d|dS)NrT) r]rrrrsrJr rrqrMrVrWrrr rPrPrQtest_broadcast_cudays   z1DistributedTest._DistTestBase.test_broadcast_cudacCs |\}}}||||dSr\)rXrqr\rPrPrQtest_broadcast_groupsz2DistributedTest._DistTestBase.test_broadcast_groupcCs |\}}}||||dSr\)rZrqr\rPrPrQtest_broadcast_full_groupsz7DistributedTest._DistTestBase.test_broadcast_full_groupz/Only NCCL backend supports high priority streamc Cs|\}}}ttt}||d}tj|tt d}|t j d<t d|t}t |\}}} t||}tj} d| _t||| | } ||| |d|ddS)Nrr?rrruFT)r]rrrrsrJr rrrrrrZ rendezvousrZ PrefixStorerOptionsZis_high_priority_streamrq) rMrVrrrr Znew_portZ gen_iteratorstorer rprWrPrPrQtest_nccl_high_priority_streams      z.r?)r]rrrPRODUCTrr~r\rPrPrQtest_reduce_productsz1DistributedTest._DistTestBase.test_reduce_productc Cs,|\}}}||||tjjddddSNir?)r]rrrMINr\rPrPrQtest_reduce_minsz-DistributedTest._DistTestBase.test_reduce_minc Cs,|\}}}||||tjjddddSNrr<)r]rrrMAXr\rPrPrQtest_reduce_maxsz-DistributedTest._DistTestBase.test_reduce_maxc Cs<|\}}}||||tjjddddt|ddSr)rXrrrrr~r\rPrPrQtest_reduce_group_sum sz3DistributedTest._DistTestBase.test_reduce_group_sumc CsF|\}}}||||tjjddtdddgt|dddS)NrAr<cSs||Sr\rPrrPrPrQr$rzIDistributedTest._DistTestBase.test_reduce_group_product..r?)rXrrrrrr~r\rPrPrQtest_reduce_group_productsz7DistributedTest._DistTestBase.test_reduce_group_productc Cs,|\}}}||||tjjddddSr)rXrrrrr\rPrPrQtest_reduce_group_min'sz3DistributedTest._DistTestBase.test_reduce_group_minc Cs,|\}}}||||tjjddddSr)rXrrrrr\rPrPrQtest_reduce_group_max0sz3DistributedTest._DistTestBase.test_reduce_group_maxc Cs<|\}}}||||tjjddddt|ddSr)rZrrrrr~r\rPrPrQtest_reduce_full_group_sum9sz8DistributedTest._DistTestBase.test_reduce_full_group_sumc CsF|\}}}||||tjjddtdddgt|dddS)NrAr<cSs||Sr\rPrrPrPrQrRrzNDistributedTest._DistTestBase.test_reduce_full_group_product..r?)rZrrrrrr~r\rPrPrQtest_reduce_full_group_productGsz.rArrzFcstdSrR)rrrP)rWr~rtensorsrPrQrs zIDistributedTest._DistTestBase._test_reduce_twice_helper..)secondary_op_callr|r?) rr rmrrr}rfr rK) rMrVrWrr~rrrr rrrqrP)rWrr~rrrrrQ_test_reduce_twice_helperfs,    z7DistributedTest._DistTestBase._test_reduce_twice_helperc Cs<|\}}}||||tjjddddt|ddSr)r]rrrrr~r\rPrPrQtest_reduce_sum_twicesz3DistributedTest._DistTestBase.test_reduce_sum_twicec Csf|\}}}ttt}||d}tj|||||tj j ddddt |dd| dSr) r]rrrrsrJr rrrrr~rsrPrPrQtest_reduce_sum_cuda_twices  z8DistributedTest._DistTestBase.test_reduce_sum_cuda_twicez#Only Nccl supports reduce_scatter_vc CsH||\}}}ttt}||d}g}|D]}||dq8t|d|}|||} t|} d} d} dD]} t| | |d}|||  | t j ||| | t j d d |}t|tt ||tjj|| }| r|ddt|d}t j ||| | t j d}| | |}|||q||dS) Nrr?rAr<TFrrr)rKr]rrrrsrrr r rJrrkr Zreduce_scatterrYrrrrr~rf)rMrVrWrrr Zinput_split_sizesrZ start_lenZend_lensum_lenrr async_valrq out_tensorrrr-rPrPrQtest_reduce_scatter_v_cudas<  $z8DistributedTest._DistTestBase.test_reduce_scatter_v_cudaTc CsX|r(|||d}|||d}|jg}|jddtj||tjj|dd|d |S)Nrz:reduce_scatter_tensorF expect_eventr|)r r}rmrZreduce_scatter_tensorrrrM tensor_out tensor_inrWrr rr|rPrPrQ_reduce_scatter_tensor_helpers" z;DistributedTest._DistTestBase._reduce_scatter_tensor_helperz-Only Nccl supports CUDA reduce_scatter_tensorc Cs|\}}}ttt}d}tj|tjd}tt ||}| ||||d|}t|||d|t |}| ||| t |t ||f}| ||||d|}| ||| dS)NrArTr?)r]rrrrsrJzerosZint64Zaranger~rrfrKr) rMrVrWrrr rrr-rPrPrQtest_reduce_scatter_tensor_cudas   z=DistributedTest._DistTestBase.test_reduce_scatter_tensor_cudac CsD|\}}}ttt}|D]}||kr>t|dd}nt|dd}|||d}t}tjj |_ |t j krt |g|}n||g|}tdkrz$|td|W5QRXWn"tk r||YnX||} n|} |ddt|d} || t|d| gq |dS)Nr?rAr<rrz0Work needs to be completed before calling result)r]rrrrsr r rrrZreduceOprr[r allreducerr,resultAssertionErrorrr\rr~rfrK) rMrVrWrrrrqrprOrrrPrPrQtest_all_reduce_result_cuda s8    z9DistributedTest._DistTestBase.test_all_reduce_result_cuda)rr profile_cudar|c s fddg} |dk r$| |tjjj|dd} | *} dd| D} |r`| D] }|qRW5QRX|rttkrt t|| }t tj j kr| t|t| |D]b}||j| |jd||jd|dk rt tj j kr| |j|d |jd |qdS) Ncs Sr\rPrPrJrSr~rPrQrB rz.T)Zuse_cudarcSsg|] }|qSrPrP)r{Zop_callrPrPrQrM sz>DistributedTest._DistTestBase.call_dist_op..r?rz event shape: z vs tensor )rrJrrrrrrPROFILING_SUPPORTED_BACKENDSrrrrrfr~rr:r8assertGreaterEqualZcpu_timer))rMZprofiling_title_postfixr:r~rrrr|rJrSZop_callsrCr+ZworksrOrrNrPrrQrm6 s@    z*DistributedTest._DistTestBase.call_dist_opc  Cs|D]} || kr|n|} t| d| d| }|rD|| |d}|jtjkr`t|jg}n|jg}|jd| t j |||| |d| dkr|rt t kr|jd| t j |||| d|d q| dS)Nr?rr :all_reduce)async_opr|T)rrr|)r r r rrJ complex64 view_as_realr}rmr all_reducer!CUDA_PROFILING_SUPPORTED_BACKENDSrK)rMrVrWrr~rrrr rrrr curr_valuerqr|rPrPrQ_test_all_reduce_helperm sH   z5DistributedTest._DistTestBase._test_all_reduce_helperc Cs<|\}}}||||tjjddddt|ddSrr]rrrrr~r\rPrPrQtest_all_reduce_sum sz1DistributedTest._DistTestBase.test_all_reduce_sumc Cs@|\}}}|j|||tjjddddt|ddddSNrAr<r?T)rrr\rPrPrQtest_all_reduce_sum_async sz7DistributedTest._DistTestBase.test_all_reduce_sum_asyncz;Only Gloo and NCCL backends will have CUDA allReduce testedc Cs\tj|j|\}}}ttt}| |||tj j ddddt |dd| dS)NrAr<r?T rJr rrr]rrrrsrrrr~rMrVrWrrrPrPrQtest_all_reduce_sum_cuda sz6DistributedTest._DistTestBase.test_all_reduce_sum_cudac Cs`tj|j|\}}}ttt}|j |||tj j ddddt |dd|dd dSrrrrPrPrQtest_all_reduce_sum_cuda_async szDistributedTest._DistTestBase.test_all_reduce_sum_cuda_complexc CsF|\}}}||||tjjddtdddgt|dddS)NrAr<cSs||Sr\rPrrPrPrQr, rzGDistributedTest._DistTestBase.test_all_reduce_product..r?)r]rrrrrr~r\rPrPrQtest_all_reduce_product" sz5DistributedTest._DistTestBase.test_all_reduce_productc Cs,|\}}}||||tjjddddSr)r]rrrrr\rPrPrQtest_all_reduce_min/ sz1DistributedTest._DistTestBase.test_all_reduce_minc Cs,|\}}}||||tjjddddSr)r]rrrrr\rPrPrQtest_all_reduce_max6 sz1DistributedTest._DistTestBase.test_all_reduce_maxc Cs<|\}}}||||tjjddddt|ddSr)rXrrrrr~r\rPrPrQtest_all_reduce_group_sum= sz7DistributedTest._DistTestBase.test_all_reduce_group_sumc CsF|\}}}||||tjjddtdddgt|dddS)NrAr<cSs||Sr\rPrrPrPrQrV rzMDistributedTest._DistTestBase.test_all_reduce_group_product..r?)rXrrrrrr~r\rPrPrQtest_all_reduce_group_productK sz;DistributedTest._DistTestBase.test_all_reduce_group_productc Cs,|\}}}||||tjjddddSr)rXrrrrr\rPrPrQtest_all_reduce_group_minY sz7DistributedTest._DistTestBase.test_all_reduce_group_minc Cs,|\}}}||||tjjddddSr)rXrrrrr\rPrPrQtest_all_reduce_group_maxa sz7DistributedTest._DistTestBase.test_all_reduce_group_maxc Cs<|\}}}||||tjjddddt|ddSr)rZrrrrr~r\rPrPrQtest_all_reduce_full_group_sumi sz.r?)rZrrrrrr~r\rPrPrQ"test_all_reduce_full_group_productv sz@DistributedTest._DistTestBase.test_all_reduce_full_group_productc Cs,|\}}}||||tjjddddSr)rZrrrrr\rPrPrQtest_all_reduce_full_group_min sz.r)r]r!rrrrrrf) rMrrVrWrtestsinputsoutputsrrPrrQ_test_sparse_all_reduce_sum s z9DistributedTest._DistTestBase._test_sparse_all_reduce_sumz+Only Gloo backend support sparse all reducecCs|dddS)NcSs|Sr\rPtrPrPrQr rzJDistributedTest._DistTestBase.test_sparse_all_reduce_sum..rrLrPrPrQtest_sparse_all_reduce_sum sz8DistributedTest._DistTestBase.test_sparse_all_reduce_sumcCs|dddS)NcSs |Sr\)cloner rrPrPrQr rzODistributedTest._DistTestBase.test_sparse_all_reduce_sum_cuda..rrLrPrPrQtest_sparse_all_reduce_sum_cuda sz=DistributedTest._DistTestBase.test_sparse_all_reduce_sum_cudacCsfddtddgddtddgdd|ddd|dtddtdd|dgtjtjtjgfS)NrArhr<rr?)rrJrkrZ group_sizerPrPrQ$_all_reduce_coalesced_sum_test_cases szBDistributedTest._DistTestBase._all_reduce_coalesced_sum_test_casescCs8ddgddgdd|ddd|dgtjtjgfS)Nr?rArhrirJrkrrPrPrQ(_all_reduce_coalesced_product_test_cases s  zFDistributedTest._DistTestBase._all_reduce_coalesced_product_test_casescCs ddgddgddgtjtjgfSNr?rirArhrrrPrPrQ$_all_reduce_coalesced_min_test_cases s  zBDistributedTest._DistTestBase._all_reduce_coalesced_min_test_casescCs ddgddgddgtjtjgfSrrrrPrPrQ$_all_reduce_coalesced_max_test_cases s  zBDistributedTest._DistTestBase._all_reduce_coalesced_max_test_casesc CsH|\}}}|td$ttdtjdgtjj |W5QRXdSr) r]rr,rall_reduce_coalescedr rJrrrr\rPrPrQ1test_all_reduce_coalesced_max_complex_unsupported szODistributedTest._DistTestBase.test_all_reduce_coalesced_max_complex_unsupportedc stjj|jtjj|jtjj|jtjj|j i|}|t |\}} } } |D]krX|n| } fddt | | D} |rfdd| D} g}| D]0}|j t jkr|t |jq||jq|jddtj| |||dfddt | | D}|| |qH|dS)Ncs"g|]\}}td||dqSr?rrd)r{rr rrPrQr szSDistributedTest._DistTestBase._test_all_reduce_coalesced_helper..csg|]}|dqSrr r{rrrrPrQr srFr{cs"g|]\}}td||dqSrrd)r{rrrrPrQr s)rrrrrrrrrrr~rhrrJrrrr}rmrrfrK)rMrVrWrr~r rZtest_case_funcZ master_valuesZ worker_valuesZexpected_valuesdtypesZ curr_valuesrr|rqr rP)rrrrQ!_test_all_reduce_coalesced_helper sP     z?DistributedTest._DistTestBase._test_all_reduce_coalesced_helpercCs,|\}}}|j|||tjjddddSNF)r r)r]rrrrr\rPrPrQtest_all_reduce_coalesced_sum sz;DistributedTest._DistTestBase.test_all_reduce_coalesced_sumcCs,|\}}}|j|||tjjddddSr)r]rrrrr\rPrPrQ!test_all_reduce_coalesced_product sz?DistributedTest._DistTestBase.test_all_reduce_coalesced_productcCs,|\}}}|j|||tjjddddSr)r]rrrrr\rPrPrQtest_all_reduce_coalesced_min& sz;DistributedTest._DistTestBase.test_all_reduce_coalesced_mincCs,|\}}}|j|||tjjddddSr)r]rrrrr\rPrPrQtest_all_reduce_coalesced_max2 sz;DistributedTest._DistTestBase.test_all_reduce_coalesced_maxcCs,|\}}}|j|||tjjddddSr)rXrrrrr\rPrPrQ#test_all_reduce_coalesced_group_sum9 szADistributedTest._DistTestBase.test_all_reduce_coalesced_group_sumcCs,|\}}}|j|||tjjddddSr)rXrrrrr\rPrPrQ'test_all_reduce_coalesced_group_productA szEDistributedTest._DistTestBase.test_all_reduce_coalesced_group_productcCs,|\}}}|j|||tjjddddSr)rXrrrrr\rPrPrQ#test_all_reduce_coalesced_group_minN szADistributedTest._DistTestBase.test_all_reduce_coalesced_group_mincCs,|\}}}|j|||tjjddddSr)rXrrrrr\rPrPrQ#test_all_reduce_coalesced_group_maxV szADistributedTest._DistTestBase.test_all_reduce_coalesced_group_maxcCs,|\}}}|j|||tjjddddSr)rZrrrrr\rPrPrQ(test_all_reduce_coalesced_full_group_sum^ szFDistributedTest._DistTestBase.test_all_reduce_coalesced_full_group_sumcCs,|\}}}|j|||tjjddddSr)rZrrrrr\rPrPrQ,test_all_reduce_coalesced_full_group_producte szJDistributedTest._DistTestBase.test_all_reduce_coalesced_full_group_productcCs,|\}}}|j|||tjjddddSr)rZrrrrr\rPrPrQ(test_all_reduce_coalesced_full_group_minq szFDistributedTest._DistTestBase.test_all_reduce_coalesced_full_group_mincCs,|\}}}|j|||tjjddddSr)rZrrrrr\rPrPrQ(test_all_reduce_coalesced_full_group_max} szFDistributedTest._DistTestBase.test_all_reduce_coalesced_full_group_maxc s|D]ĉtddd}tdd}krHfdd|Dng} |rv|d}fdd| D} tjkrdd| D} nd d| D} |jd d tj|| |d | d |||q|dS) Nr?rrcsg|]}td|dqSrrdrrYrrPrQr szFDistributedTest._DistTestBase._test_scatter_helper..rcsg|]}|dqSrrrrrPrQr scSsg|]}t|jqSrP)rJrr}rrPrPrQr scSsg|] }|jqSrP)r}rrPrPrQr sz:scatterF)r scatter_listrVrr|) r r rJrrmrscatterrfrK) rMrVrWrr rrrqr-rr|rPrYrrrrQ_test_scatter_helper s4  z2DistributedTest._DistTestBase._test_scatter_helperZuccz&CPU tensor ops not supported by UCP TLcs|\}}}tdgd}|dkrRfdd|D}tj|d|dntj|dd|||d}|dkrfdd|D}tj||d n t||||dS) Nr?rrcsg|]}|qSrPrrZonerPrQr szEDistributedTest._DistTestBase.test_scatter_checks..)rrrcsg|]}|qSrPr rr rPrQr s)r)r]rJrrrrrf)rMrVrWroutputrrPr rQtest_scatter_checks s    z1DistributedTest._DistTestBase.test_scatter_checkscCs |\}}}||||dSr\)r]rr\rPrPrQ test_scatter sz*DistributedTest._DistTestBase.test_scatterzOnly Nccl supports CUDA gathercCs2|\}}}ttt}||||d|dSNT)r]rrrrsrrrPrPrQtest_scatter_cuda sz/DistributedTest._DistTestBase.test_scatter_cudacCs&|\}}}|j|||tjddSr)r]rrJrr\rPrPrQtest_scatter_complex sz2DistributedTest._DistTestBase.test_scatter_complexcCs8|\}}}ttt}|j|||d|tjddSNTr)r]rrrrsrrJrrrPrPrQtest_scatter_cuda_complex sz7DistributedTest._DistTestBase.test_scatter_cuda_complexcCs |\}}}||||dSr\)rXrr\rPrPrQtest_scatter_group sz0DistributedTest._DistTestBase.test_scatter_groupcCs |\}}}||||dSr\)rZrr\rPrPrQtest_scatter_full_group sz5DistributedTest._DistTestBase.test_scatter_full_groupc s|D]ʉtd}kr0fdd|Dng}|r^|d}fdd|D}|jddtj|||dt|dkr|djgndd krfd d|D}t||D]\} } || | qq| dS) Nr?csg|]}tddqS)r?rrdrrYrPrQr szEDistributedTest._DistTestBase._test_gather_helper..rcsg|]}|dqSrrrrrPrQr sz:gatherF)r, gather_listrVrr|csg|]}td|qSrfrdrrrPrQr s) r r rmrgatherr~r}rhrfrK) rMrVrWrr rrqrr t1t2rP)rYrrrQ_test_gather_helper s. z1DistributedTest._DistTestBase._test_gather_helpercs|\}}}tdg|dkrhfdd|D}tj|d|d|D]}||||qLntj|dd|dkrƇfdd|D}tj||d|D]}||||qnt|dS) Nr?rcsg|] }qSrPr rr rPrQr szDDistributedTest._DistTestBase.test_gather_checks..)r,rr,csg|] }qSrPr rr rPrQr s)r)r]rJrrrrf)rMrVrWrrrrPr rQtest_gather_checks s z0DistributedTest._DistTestBase.test_gather_checkscCs |\}}}||||dSr\)r]rr\rPrPrQ test_gather sz)DistributedTest._DistTestBase.test_gathercCs2|\}}}ttt}||||d|dSr)r]rrrrsrrrPrPrQtest_gather_cuda sz.DistributedTest._DistTestBase.test_gather_cudacCs |\}}}||||dSr\)rXrr\rPrPrQtest_gather_group& sz/DistributedTest._DistTestBase.test_gather_groupcCs |\}}}||||dSr\)rZrr\rPrPrQtest_gather_full_group- sz4DistributedTest._DistTestBase.test_gather_full_groupc s|D]؉tdd}fdd|D}tj} |r^|d}fdd|D}|djtjkrt|djg} n |djg} |j dd| |||d| d fd d|D} t || D]\} } | | | qq| dS) Nr?rcsg|]}tdddqS)r?rrrdrrrPrQr9 szIDistributedTest._DistTestBase._test_all_gather_helper..rcsg|]}|dqSrrrrrPrQr= s :all_gatherFr{csg|]}td|dqSrrdrrrPrQrM s) r rrir rrJrrr}rmrhrfrK)rMrVrWrr rrrqrZ allgatherr|r rrrPrrQ_test_all_gather_helper4 s4  z5DistributedTest._DistTestBase._test_all_gather_helpercCs |\}}}||||dSr\)r]r"r\rPrPrQtest_all_gatherU sz-DistributedTest._DistTestBase.test_all_gatherz"Only Nccl supports CUDA all gathercCs2|\}}}ttt}||||d|dSr)r]rrrrsr"rrPrPrQtest_all_gather_cudaZ sz2DistributedTest._DistTestBase.test_all_gather_cudacCs&|\}}}|j|||tjddSr)r]r"rJrr\rPrPrQtest_all_gather_complexa sz5DistributedTest._DistTestBase.test_all_gather_complexcCs8|\}}}ttt}|j|||d|tjddSr)r]rrrrsr"rJrrrPrPrQtest_all_gather_cuda_complexf sz:DistributedTest._DistTestBase.test_all_gather_cuda_complexcCs |\}}}||||dSr\)rXr"r\rPrPrQtest_all_gather_groupo sz3DistributedTest._DistTestBase.test_all_gather_groupcCs |\}}}||||dSr\)rZr"r\rPrPrQtest_all_gather_full_groupu sz8DistributedTest._DistTestBase.test_all_gather_full_groupzOnly Nccl supports all_gather_vcCs||\}}}ttt}||d}g}|D]}||dq8t|}d} dD]|} tj ||||tj d |  |} t |d|d} ttt| || || } | r| | }t |||d}|| |q\|dS)Nrr?rArrrr)rKr]rrrrsrrrJrrkr r r rirYrrrf)rMrVrWrrr Zoutput_split_sizesr,rrarrqrrrr-rPrPrQtest_all_gather_v_cudaz s0 $z4DistributedTest._DistTestBase.test_all_gather_v_cudac Csn|r(|||d}|||d}|jtjkrDt|jg}n|jg}|jddtj|||dd|d |S)Nrz:all_gather_into_tensorFr) r rrJrrr}rmrZall_gather_into_tensorrrPrPrQ_all_gather_into_tensor_helper s$  z.) r]rrrrsrJrr~r*rKrfrKrMrVrWrrrrr-rPr,rQ$test_all_gather_into_cat_tensor_cuda s zBDistributedTest._DistTestBase.test_all_gather_into_cat_tensor_cudacs|\}}}ttt}dtg|}tt|gd}|||||d|}t fdd|D}| ||| dS)NrArTcsg|]}tg|qSrPr+rr,rPrQr szXDistributedTest._DistTestBase.test_all_gather_into_stack_tensor_cuda..) r]rrrrsrJrr~r*stackrfrKr-rPr,rQ&test_all_gather_into_stack_tensor_cuda s zDDistributedTest._DistTestBase.test_all_gather_into_stack_tensor_cudac Csg}|D]0}|jtjkr,|t|jq||jq|jddtj||||dt ||D]0\}}t ||D]\} } t | | spdSqpq^dS)z| Helper that runs all_gather_coalesced and returns true if output matches expectations. r!Fr{T) rrJrrrr}rmrZall_gather_coalescedrhr`) rMoutput_tensor_lists input_tensorsr rWr| input_tensorl1l2rrrPrPrQ$_run_all_gather_coalesced_and_verify s&    zBDistributedTest._DistTestBase._run_all_gather_coalesced_and_verifycs~|dk rrtddD]^fddtdD}fdd|D}fdd|D}|||||stdq|dS) NrArncs g|]}t|||dqSrrr{Z tensor_id)rrrPrQr szSDistributedTest._DistTestBase._test_all_gather_coalesced_helper..r?cs$g|]}fddtdDqS)csg|]}t||ddqS)rrr8r9rrPrQr s^DistributedTest._DistTestBase._test_all_gather_coalesced_helper...r?rrr test_case_idrPrQr s cs&g|]fddtdDqS)cs g|]}t|||dqSr7r8r9)r rank_iterrPrQr sr:r?r;)r{r<)r>rQr s z+output tensors do not match expected ouputs)rr6rrK)rMrVrWrrr2r1r rP)rrr=rQ!_test_all_gather_coalesced_helper s(    z?DistributedTest._DistTestBase._test_all_gather_coalesced_helperZallgather_coalescedz& does not support all_gather_coalescedcCs |\}}}||||dSr\)r]r?r\rPrPrQ test_all_gather_coalesced_simple sz>DistributedTest._DistTestBase.test_all_gather_coalesced_simplecCs&|\}}}|j|||tjddSr)r]r?rJrr\rPrPrQ!test_all_gather_coalesced_complex& sz?DistributedTest._DistTestBase.test_all_gather_coalesced_complexcCs |\}}}||||dSr\)rXr?r\rPrPrQtest_all_gather_coalesced_group0 sz=DistributedTest._DistTestBase.test_all_gather_coalesced_groupcCs |\}}}||||dSr\)rZr?r\rPrPrQ$test_all_gather_coalesced_full_group9 szBDistributedTest._DistTestBase.test_all_gather_coalesced_full_groupcCs|\}}}|tddgtdg|dtddgtdgtdgg}dd|D}dd|D}|||||st|dS)NrArr?rhc SsZg|]R}dtddgdtdgdtddgdtdgdtdggqS)rrArrhr+rrPrPrQrN szVDistributedTest._DistTestBase.test_all_gather_coalesced_with_empty..c SsRg|]J}|tddgtdg|dtddgtdgtdggqS)rArr?rhr+r[rPrPrQrX s   )r]rJrr6rrK)rMrVrWrr2Zoutput_tensors_listsr rPrPrQ$test_all_gather_coalesced_with_emptyA s(     zBDistributedTest._DistTestBase.test_all_gather_coalesced_with_emptyc s|dk rt|tjgd|}tfdd|D}tjgdd} |r|||d}|||d}| ||d} tjkrt|jg} n|jg} |jddt j | ||| d| | || dS) Nrcs"g|]}tjdgd|qSrr+rrr rPrQro sz\DistributedTest._DistTestBase._test_all_to_all_single_equal_split_helper..rrz :all_to_allF)rVr|) r~rJrrKr rrr}rmrall_to_all_singlerfrK) rMrVrWrr rr in_tensorr-rr|rPrErQ*_test_all_to_all_single_equal_split_helperh s2  zHDistributedTest._DistTestBase._test_all_to_all_single_equal_split_helperc s|dk rt|dd|D}fdd|D}tjt|gd} tjdgd} tfdd|D} |r| |d} | |d} | |d} tj| | |||d|| | | dS) NcSsg|] }|dqSrfrPrrPrPrQr sz^DistributedTest._DistTestBase._test_all_to_all_single_unequal_split_helper..csg|] }dqSrfrPr)rrPrQr srr?cs&g|]}tjdgd|qSrr+rrrr rPrQr srrV) r~rJrrrKr rrFrfrK) rMrVrWrr rr in_splitsZ out_splitsrGrr-rPrIrQ,_test_all_to_all_single_unequal_split_helper s, zJDistributedTest._DistTestBase._test_all_to_all_single_unequal_split_helperc s|dk rt|dd|Dfddt|D}fdd|D}fdd|D} |rfdd|D}fdd| D} fdd|D}tj|||d t|| D]\} } || | q|dS) NcSsg|] }|dqSrfrPrrPrPrQr szIDistributedTest._DistTestBase._test_all_to_all_helper..cs*g|]"\}}tj|gdqSr7r+)r{rr)rrKrr rPrQr scs"g|]}tjdgdqSrr+rrIrPrQr scs&g|]}tjdgd|qSrr+rrIrPrQr scsg|]}|dqSrrrrrPrQr scsg|]}|dqSrrrrrPrQr scsg|]}|dqSrrrrrPrQr srJ)r~ enumeraterZ all_to_allrhrfrK) rMrVrWrr rrZ in_tensorsZ out_tensorsr rrrP)rrKrrr rQ_test_all_to_all_helper s,  z5DistributedTest._DistTestBase._test_all_to_all_helpermpiz'Only MPI supports CPU all_to_all_singlecCs |\}}}||||dSr\)r]rHr\rPrPrQ"test_all_to_all_single_equal_split sz@DistributedTest._DistTestBase.test_all_to_all_single_equal_splitz)Only Nccl supports CUDA all_to_all_singlecCs2|\}}}ttt}||||d|dSr)r]rrrrsrHrrPrPrQ'test_all_to_all_single_equal_split_cuda szEDistributedTest._DistTestBase.test_all_to_all_single_equal_split_cudacCs&|\}}}|j|||tjddSr)r]rHrJrr\rPrPrQ*test_all_to_all_single_equal_split_complex szHDistributedTest._DistTestBase.test_all_to_all_single_equal_split_complexcCs8|\}}}ttt}|j|||d|tjddSr)r]rrrrsrHrJrrrPrPrQ/test_all_to_all_single_equal_split_cuda_complex szMDistributedTest._DistTestBase.test_all_to_all_single_equal_split_cuda_complexcCs |\}}}||||dSr\)r]rLr\rPrPrQ$test_all_to_all_single_unequal_split szBDistributedTest._DistTestBase.test_all_to_all_single_unequal_splitcCs2|\}}}ttt}||||d|dSrr]rrrrsrLrrPrPrQ)test_all_to_all_single_unequal_split_cuda szGDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_cudacCs&|\}}}|j|||tjddSr)r]rLrJrr\rPrPrQ,test_all_to_all_single_unequal_split_complex szJDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_complexcCs8|\}}}ttt}|j|||d|tjddSr)r]rrrrsrLrJrrrPrPrQ1test_all_to_all_single_unequal_split_cuda_complex szODistributedTest._DistTestBase.test_all_to_all_single_unequal_split_cuda_complexzOnly MPI supports all_to_allcCs |\}}}||||dSr\)r]rNr\rPrPrQtest_all_to_allsz-DistributedTest._DistTestBase.test_all_to_allz"Only NCCL supports CUDA all_to_allcCs2|\}}}ttt}||||d|dSr)r]rrrrsrNrrPrPrQtest_all_to_all_cuda sz2DistributedTest._DistTestBase.test_all_to_all_cudacCs&|\}}}|j|||tjddSr)r]rNrJrr\rPrPrQtest_all_to_all_complexsz5DistributedTest._DistTestBase.test_all_to_all_complexcCs8|\}}}ttt}|j|||d|tjddSr)r]rrrrsrNrJrrrPrPrQtest_all_to_all_cuda_complexsz:DistributedTest._DistTestBase.test_all_to_all_cuda_complexcCs |\}}}||||dSr\)rXrHr\rPrPrQ(test_all_to_all_single_equal_split_group szFDistributedTest._DistTestBase.test_all_to_all_single_equal_split_groupcCs2|\}}}ttt}||||d|dSr)rXrrrrsrHrrPrPrQ-test_all_to_all_single_equal_split_group_cuda&szKDistributedTest._DistTestBase.test_all_to_all_single_equal_split_group_cudacCs |\}}}||||dSr\)rXrLr\rPrPrQ*test_all_to_all_single_unequal_split_group4szHDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_groupcCs2|\}}}ttt}||||d|dSrrUrrPrPrQ/test_all_to_all_single_unequal_split_group_cuda:szMDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_group_cudacCs |\}}}||||dSr\)rXrNr\rPrPrQtest_all_to_all_groupHsz3DistributedTest._DistTestBase.test_all_to_all_groupcCs2|\}}}ttt}||||d|dSr)rXrrrrsrNrrPrPrQtest_all_to_all_group_cudaNsz8DistributedTest._DistTestBase.test_all_to_all_group_cudacCs |\}}}||||dSr\)rZrHr\rPrPrQ-test_all_to_all_single_equal_split_full_groupVszKDistributedTest._DistTestBase.test_all_to_all_single_equal_split_full_groupcCs2|\}}}ttt}||||d|dSr)rZrrrrsrHrrPrPrQ2test_all_to_all_single_equal_split_full_group_cuda[szPDistributedTest._DistTestBase.test_all_to_all_single_equal_split_full_group_cudacCs |\}}}||||dSr\)rZrLr\rPrPrQ/test_all_to_all_single_unequal_split_full_grouphszMDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_full_groupcCs2|\}}}ttt}||||d|dSr)rZrrrrsrLrrPrPrQ4test_all_to_all_single_unequal_split_full_group_cudamszRDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_full_group_cudacCs |\}}}||||dSr\)rZrNr\rPrPrQtest_all_to_all_full_groupzsz8DistributedTest._DistTestBase.test_all_to_all_full_groupcCs2|\}}}ttt}||||d|dSr)rZrrrrsrNrrPrPrQtest_all_to_all_full_group_cudasz=DistributedTest._DistTestBase.test_all_to_all_full_group_cudac Csd}|D]}tdd}|r2|||d}||krt|t|t|||t|dt|qt|||t|| t tt |dd||fdq|j dd dS) N333333?r?rr'z!destination rank: %d, my rank: %dz3 (if you see this failure, please report in #14554)rr) rJZ DoubleTensorr r r*rrr-rrrkrK) rMrVrWrr rZ WAIT_TIMErYrrPrPrQ_test_barrier_helpers(      z2DistributedTest._DistTestBase._test_barrier_helperz MPI doesn't supports GPU barrierz flaky on PyTorch CI with timeoutcCs2|\}}}ttt}||||d|dSr)r]rrrrsrkrrPrPrQtest_barrier_cudasz/DistributedTest._DistTestBase.test_barrier_cudacCs2|\}}}ttt}||||d|dSr)rXrrrrsrkrrPrPrQtest_barrier_group_cudasz5DistributedTest._DistTestBase.test_barrier_group_cudacCs2|\}}}ttt}||||d|dSr)rZrrrrsrkrrPrPrQtest_barrier_full_group_cudasz:DistributedTest._DistTestBase.test_barrier_full_group_cudaz cpu barrierz does not support CPU barriercCs |\}}}||||dSr\)r]rkr\rPrPrQ test_barriersz*DistributedTest._DistTestBase.test_barriercCs |\}}}||||dSr\)rXrkr\rPrPrQtest_barrier_groupsz0DistributedTest._DistTestBase.test_barrier_groupcCs |\}}}||||dSr\)rZrkr\rPrPrQtest_barrier_full_groupsz5DistributedTest._DistTestBase.test_barrier_full_groupcs||D]jtd}fdd||D}|krJ|j||dd|d<t|||D]}|||q\q|dS)Nr?cs"g|]}tddj|dqS)r?rrr r rrrPrQrszQDistributedTest._DistTestBase._test_broadcast_multigpu_helper..rr)r r rZbroadcast_multigpurfrK)rMrVrWrrr-rrqrPrrQ_test_broadcast_multigpu_helpers  z=DistributedTest._DistTestBase._test_broadcast_multigpu_helperz&MPI doesn't support broadcast multigpuzNCCL broadcast multigpu skippedcCs0|\}}}ttt}|||||dSr\)r]rrrrsrsrrPrPrQtest_broadcast_multigpusz5DistributedTest._DistTestBase.test_broadcast_multigpuc s~|D]l|kr|n|fdd||D} |ddtj| ||td|d} | D]} || | q^q|dS)Ncs&g|]}tddj|dqSr?rrrrrrrrrPrQrszRDistributedTest._DistTestBase._test_all_reduce_multigpu_helper..rFr?r)rmrZall_reduce_multigpur rfrK) rMrVrWrrr~rrrrrr-rqrPrvrQ _test_all_reduce_multigpu_helpers" z>DistributedTest._DistTestBase._test_all_reduce_multigpu_helperz)CUDA all_reduce multigpu skipped for NCCLzUCC all_reduce multigpu skippedc CsX|\}}}ttt}|||||tjjddddt|dt|ddS)NrAr<r?r) r]rrrrsrwrrr~rrPrPrQtest_all_reduce_multigpusz6DistributedTest._DistTestBase.test_all_reduce_multigpuc Csv|\}}}ttt}|j||||tjjtddtddtddtddt |dt |dt j d dS)NrArhr<rr?rr) r]rrrrsrwrrrr~rJrrrPrPrQ test_all_reduce_multigpu_complexs z>DistributedTest._DistTestBase.test_all_reduce_multigpu_complexc s|D]|kr|n|fdd||D} |jddtj| ||t| dk| djgd |krtd|} || d| q|dS)Ncs"g|]}tdj|dqS)r?rrrrrZ tensor_valuerPrQr7szNDistributedTest._DistTestBase._test_reduce_multigpu_helper..rzFr?rr)rmrZreduce_multigpur~r}r rfrK) rMrVrWrrr~rrrrr-rPrzrQ_test_reduce_multigpu_helper*s(     z:DistributedTest._DistTestBase._test_reduce_multigpu_helperz*Only Nccl backend supports reduce multigpuc Csp|\}}}ttt}||d}tj||||||tj j ddddt |dt |ddS)NrrAr<r?) r]rrrrsrJr rr{rrr~rsrPrPrQtest_reduce_multigpuLs  z2DistributedTest._DistTestBase.test_reduce_multigpuc s|D]މfdd||D}g}g}tdddgt|dt|} tddgt|dt|} ||D]4|fdd| D|fdd| Dq~|jd d tj|||t|dkd |||q|dS) Ncs$g|]}tddj|dqSrurrrrrPrQrdszRDistributedTest._DistTestBase._test_all_gather_multigpu_helper..r?rrrcsg|]}|jdqSrrrrrPrQrxscsg|]}|jdqSr}rrr~rPrQrzsr!F)r)r r~rrmrZall_gather_multigpurfrK) rMrVrWrrrrZoutput_tensorsZexpected_outputZoutput_per_gpuZexpected_per_gpurP)rYrrrQ _test_all_gather_multigpu_helper`sD      z>DistributedTest._DistTestBase._test_all_gather_multigpu_helperz-Only Nccl backend supports allgather multigpucCsH|\}}}ttt}||d}tj||||||dSr) r]rrrrsrJr rrrsrPrPrQtest_all_gather_multigpus   z6DistributedTest._DistTestBase.test_all_gather_multigpucCsN|\}}}ttt}||d}tj||j||||tj ddS)Nrr) r]rrrrsrJr rrrrsrPrPrQ test_all_gather_multigpu_complexs  z>DistributedTest._DistTestBase.test_all_gather_multigpu_complexc Cs@|D]2}|jdk rt||j7}W5QRXd|_qdSr\)rrrJno_gradrMrrrPrPrQ _model_steps    z)DistributedTest._DistTestBase._model_stepc CsP|D]B}|jdk rt||j7}W5QRX|jd|jqdSNF)rrrJrrequires_grad_zero_rrPrPrQ_model_step_with_zero_grads     z8DistributedTest._DistTestBase._model_step_with_zero_gradcCsBttjd}||}t|d}t|d}t}||||fS)NrrAri)rrrrJrKrEMSELoss)rMlocal_bsr global_bs input_cputargetrrPrPrQ_prepare_dummy_datas   z1DistributedTest._DistTestBase._prepare_dummy_data?c CsD|||}||||}||dk r@||j|ddS)N memory_format)trainbackwardrZ is_contiguous) rMr input_varrrZ scale_factorrr lrPrPrQ_test_DDP_helpers z.DistributedTest._DistTestBase._test_DDP_helpercCs8|t|t|t||D]\}}|||qdSr\)rfr~rh)rMZ param_gpuZ param_DDPZp_gpuZp_DDPrPrPrQ_assert_equal_paramsz1DistributedTest._DistTestBase._assert_equal_paramrrnc Cst|D]6}|j||||| d| dkr2||} |j||| | ||| | ||| dkrj| ||nd| d| r||||n|||||t|t|j|t |}| r|dkrt drt N}tjdkrt|||dt|}nt||jt|j}W5QRXqt &}t|||dt|}W5QRX|D]"}|||||q~dS)Nrrr?rArr9)rrrrrrYrrrJrandpermrt startswithrNamedTemporaryFilerrsaveseekloadrx TemporaryFile state_dictrf)rM model_base model_DDPrrrrr batch_sizeZ test_saveoffsetr zero_gradrZn_iteridxtmptmp_fileZ saved_modelr^rPrPrQ_test_DDP_nitersT               z-DistributedTest._DistTestBase._test_DDP_niterc Cst}t|}||dt|} | |dtjj| |||d} |rT| t L} t j dkrt | | | dt | } nt | | jt | j} W5QRXt|} || \} } }}||| | |d||d|| || d |dS)Nr device_idsgradient_as_bucket_view static_graphr9T)DDP_NETrrr rEparallelr_set_static_graphrrrrrJrrrrxr~rrrK)rM gpu_subsetr output_devicerrset_static_graph_twicer model_gpurrrrrrrrPrPrQ_test_DistributedDataParallel sD           z;DistributedTest._DistTestBase._test_DistributedDataParallelc Csl|\}}}t}t|}tjj||d}d}||\}} } } |j||| | | |||ddd | |S)NrrAFT)r) r]rrrrErrrrrK) rMrrVrWrrrrrrrrrPrPrQ _test_DistributedDataParallelCPUIs.  z>DistributedTest._DistTestBase._test_DistributedDataParallelCPUz'nccl does not support DDP on CPU modelscCs |dSr\rrLrPrPrQtest_DistributedDataParallelCPUlsz=DistributedTest._DistTestBase.test_DistributedDataParallelCPUcCs|jdddS)NTrrrLrPrPrQ,test_DistributedDataParallelCPU_grad_is_viewpszJDistributedTest._DistTestBase.test_DistributedDataParallelCPU_grad_is_viewddpz1 backend does not support DistributedDataParallelcCs|tdd|dS)NcSstjtSr\)rErrrrPrPrPrQr{rzZDistributedTest._DistTestBase.test_DistributedDataParallel_requires_grad..)rr,rKrLrPrPrQ*test_DistributedDataParallel_requires_gradts zHDistributedTest._DistTestBase.test_DistributedDataParallel_requires_gradzGloo-only testcCsbGdddtj}|}tjj|}tdD].}|jdd|tdd| Dq.dS)Ncs$eZdZfddZddZZS)zBDistributedTest._DistTestBase.test_ddp_create_graph..Modelcs tttd|_dS)Nr)rCrDrErrJrqrrLrNrPrQrDs zKDistributedTest._DistTestBase.test_ddp_create_graph..Model.__init__cSs |jdSr)rpowrLrPrPrQrUszJDistributedTest._DistTestBase.test_ddp_create_graph..Model.forwardrVrPrPrNrQModels rT)Z create_graphcSsg|] }|jqSrPr)r{rrPrPrQrszGDistributedTest._DistTestBase.test_ddp_create_graph..) rErrJrrrrrrr)rMrr ddp_modelrrPrPrQtest_ddp_create_graphs z3DistributedTest._DistTestBase.test_ddp_create_graphrc Cs,tj|j}|j}tj|tjjjtjjdddd||gd}t dD]}|j j j }|dk r|| d|t|g|}||}||j j j }|}t|ttjd} || tddt | D| } |j|d | d | d |d |jd qTW5QRXdS)Nr?Fr=rrrcss|] }|VqdSr\rPrrPrPrQrsz`DistributedTest._DistTestBase.test_DistributedDataParallel_non_default_stream..rrzExpected gradient of but got z on rank msg)rJr ZStreamrstreamrErrrFrrweightrrrrqrkrrrrrrrrZdiv_rf) rMrrnetrrbatchravgr expected_gradrPrPrQ/test_DistributedDataParallel_non_default_streams4       zMDistributedTest._DistTestBase.test_DistributedDataParallel_non_default_streamr z@ backend does not support DDP communication hook on CUDA devicesc Cstjtjtjtjtjtjg}t j j t j j g}|D]r}t jjjt jjdddd|j|jgd}|}||dd|d||}||d|jq0|D]r}t jjjt jjdddd|j|jgd}|}||dd|||}||dt|qt jjjt jjdddd|j|jgd}|}||ddtdD],}t jdd|jd}||}|qj|}||ddddS) Nr?Fr=r comm_hookrAr)defaultallreduce_hookZfp16_compress_hookpowerSGDrZbatched_powerSGD_hookquantization_hooksZquantization_pertensor_hookZquantization_perchannel_hookrZBuiltinCommHookTypeZ ALLREDUCEZ FP16_COMPRESSrJrErrrFr r_get_ddp_logging_datarfrGregister_comm_hookrYZ_register_builtin_comm_hookrrrrr) rMhooksZcpp_builtin_hooksrrddp_logging_datarrrrPrPrQtest_ddp_comm_hook_loggingsT      z8DistributedTest._DistTestBase.test_ddp_comm_hook_loggingc Os|j}tj|t|tj|ttddfg}trh|t j tddddf|D]B\} } tj j jddddtjjjt| |jg||d} tjjjt| |jg||d} | } | }|r t| } t|}|t| d| dg} |dg}|rB| j|f|d | i|n| j|f||||f||}t| | D]\}}|||qvtt| }td D]&}| | | }|}|qt td D].}| | | }|}||!qt t| | D]\}}|||q*|r|"|dt| d||ddt| ddn|"|t| t W5QRXqldS) Nr?rrhTF)ZenabledZ deterministicZ benchmarkrrZ optim_paramsr)#rrJr r manual_seedrrKHAS_TORCHVISIONr torchvisionmodelsresnet50rcudnnflagsrErrrrrrYZ assertGreaterr~Z_register_fused_optimrhrfrrrrrrrassertNotEqual)rMgrad_as_bucket_viewrZ optim_clsoptimize_subsetZfunctional_optim_argsZfunctional_optim_kwargsrmodels_to_testrrZddp_model_with_optimizer_hookZddp_model_with_no_hookZ hook_paramsZno_hook_paramsZoptimizer_no_hookZ hook_paramZallreduce_paramZopt_hook_init_paramsrrvrrPrPrQ$_test_ddp_hook_with_optimizer_paritys                  zBDistributedTest._DistTestBase._test_ddp_hook_with_optimizer_parityzUIssues with async error handling, see https://github.com/pytorch/pytorch/issues/73259rrrc Cs,d}d}d}|j||tjj||||ddS)N{Gz??rư>ZbetasZeps)rrJrZAdamW)rMrrrZadamw_lrZ adamw_betasZ adamw_epsrPrPrQ)test_ddp_hook_with_optimizer_parity_adamwszGDistributedTest._DistTestBase.test_ddp_hook_with_optimizer_parity_adamwc Cs,d}d}d}|jddtjj||||ddS)NrrrTFr)rrJrZAdam)rMrZadam_lrZ adam_betasZadam_epsrPrPrQ(test_ddp_hook_with_optimizer_parity_adamszFDistributedTest._DistTestBase.test_ddp_hook_with_optimizer_parity_adamc Cs,d}d}d}|jddtjj||||ddS)NrrTF)rZ weight_decay)rrJrr)rMrZsgd_lrZ sgd_momentumZsgd_weight_decayrPrPrQ'test_ddp_hook_with_optimizer_parity_sgdszEDistributedTest._DistTestBase.test_ddp_hook_with_optimizer_parity_sgdrc Cs|j}tjdd}z |j}Wntk r6|}YnXtjjjt | ||g|d}|j ||dtjjjt | ||g|d}t dD]} |j jj|j jjfD]} | dk r| d| qt|g|} || } | |j jj} | }tddt tDt}|| }||j jj}|}| |kr|j|d |d |d |d d |j|d |d d |d d|d d qdS)Nr?rnrrrrrFcss|] }|VqdSr\rPrrPrPrQrszFDistributedTest._DistTestBase._test_ddp_hook_parity..rzExpected hook grad of rrz,Expected hook grad to be close to allreduce z , but got )rrJrErFrAttributeErrorrrrrrrrrrrrrrqrkr rrrrrrfitem)rMrrnum_validated_itersrmrZ net_with_hookZnet_without_hookrgrrrrr loss_hookZ grad_hookZavg_hookrPrPrQ_test_ddp_hook_paritys\          z3DistributedTest._DistTestBase._test_ddp_hook_paritycCs|jdtjddS)Nr)rrrrLrPrPrQtest_ddp_hook_parity_allreducesz.r) rrrrsrrJ distributedrTrrr)rMrgpusrrPrrQ,test_ddp_hook_parity_allreduce_process_groups   zJDistributedTest._DistTestBase.test_ddp_hook_parity_allreduce_process_groupcCs0dD]&}tjddd|d}|j|tjdqdS)Nrr?rA)rmatrix_approximation_rankstart_powerSGD_iter warm_startr)r PowerSGDStaterr)rMrpowersgd_staterPrPrQtest_ddp_hook_parity_powerSGDsz;DistributedTest._DistTestBase.test_ddp_hook_parity_powerSGDzmDisabled for environments that don't support multiprocessing with spawn start methodcCstjdtjjdd}|j|tjdd}tjdtjj|dd}|j|tj|d|jtkr~tjdddd}|j|tjdtjdddd}|j|tjddS)Nr<)rrstart_localSGD_iterrF)rrrZpost_local_gradient_allreduce)rrrr) post_localSGDZPostLocalSGDStaterrVr[rrrr)rMrrrPrPrQ"test_ddp_hook_parity_post_localSGD#sP z@DistributedTest._DistTestBase.test_ddp_hook_parity_post_localSGDc Csxt}|r|dn td|}tt||||d|d} ||t|d|} t|d|} || | | fS)Nrcuda:%dMbP?)rr bucket_cap_mbrrAri)rrJrrrrrrK) rMrrdevicesrglobal_batch_sizerrrrrrrPrPrQ_prepare_single_device_moduleYs  z;DistributedTest._DistTestBase._prepare_single_device_modulecCs@t}tt||d|d}t|d}t|d}||||fS)Nr)rrrrAri)rrrrrJrK)rMrrrrrrrrPrPrQ_prepare_cpu_moduless  z1DistributedTest._DistTestBase._prepare_cpu_modulec Cs|\}}}t}tdks$tdkrB|}d} ||||\} } } } tdkrttt}||dd}dd|D}|}t|} |||||||\} } } } |dk r| ||dd }t *| | | | W5QRXW5QRXt |D]}|| | | | || |d| }| || |d| }|d d krj| || ||W5QRXn || ||t| | D]F\}}|jsq|d d kr||j|jn||j|jqt d || t |} qdS) a# This is the recommended way to implement accumulate grads. If ``ddp_comm_hook`` input was specified, it will also register that hook to the ``ddp_model``. The hook fed into this function should not change the resulting gradients. rOrr?r?NcSsg|]}tdt|qSzcuda:rJrrrrPrPrQrszTDistributedTest._DistTestBase._test_accumulate_gradients_no_sync..cSs0|||}t|||j}|dSr\)rrmse_lossrrr)rrrr rrPrPrQ step_modelszTDistributedTest._DistTestBase._test_accumulate_gradients_no_sync..step_modelrAr9)r]rrsr rrr~rrrJrno_syncrrrhrrrrrfrr)rM num_iters ddp_comm_hookrrVrWrrrZlocal_batch_sizerrrrrZ int_devicesrr  iterationZ ddp_inputZ ddp_targetrjrPrPrQ"_test_accumulate_gradients_no_syncsj            z@DistributedTest._DistTestBase._test_accumulate_gradients_no_syncz2get_future is only supported on mpi, nccl and gloocCs |dS)V Runs _test_accumulate_gradients_no_sync using default inputs NrrLrPrPrQ!test_accumulate_gradients_no_syncs z?DistributedTest._DistTestBase.test_accumulate_gradients_no_synccCs|jdddS)rTrNrrLrPrPrQ.test_accumulate_gradients_no_sync_grad_is_views zLDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_grad_is_viewcs:tttjtjjtjdfdd }|jd|ddS)z Runs multiple iterations on _test_accumulate_gradients_no_sync using allreduce hook and validates whether future result was properly passed as gradients in reducer. rWbucketreturncs&|g}||ddS)NcSs |dSrrfutrPrPrQrrzxDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_hook..allreduce_hook..)r@r get_futurethen)rWrrrrPrQrs zfDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_hook..allreduce_hookrirrN robjectrZ GradBucketrJfuturesZFuturer_r)rMrrPrrQ0test_accumulate_gradients_no_sync_allreduce_hooks   zNDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_hookcs:tttjtjjtjdfdd }|jd|ddS)ao Runs multiple iterations on _test_accumulate_gradients_no_sync using allreduce hook that also uses then callbacks. In first then callback result is multiplied by 2, and the second callback divides the result by 2 * world_size. It validates whether final result was properly passed as gradients in reducer. rcs8||g}dd}fdd}|||S)NcSsd|dS)NrArrrrPrPrQmultszDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_with_then_hook..allreduce_with_then_hook..multcs|dSrr%rrrPrQdivszDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_with_then_hook..allreduce_with_then_hook..div)rr@rr)rWrrr&r'rrPrQallreduce_with_then_hooks zzDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_with_then_hook..allreduce_with_then_hookrir Nr!)rMr(rPrrQ:test_accumulate_gradients_no_sync_allreduce_with_then_hooks  zXDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_with_then_hookc Csdd}dd}|\}}}tdd}tdkrTttt}||d}||}||g} | | | } tddt |dd } | | d| dS) NcSsdd|DS)NcSsg|] }|dqSrZrPrrPrPrQr0szODistributedTest._DistTestBase.test_get_future..mult..r%rrPrPrQr&/sz;DistributedTest._DistTestBase.test_get_future..multcSsdd|DS)NcSsg|] }|dqSrfrPrrPrPrQr3szNDistributedTest._DistTestBase.test_get_future..add..r%rrPrPrQr}2sz:DistributedTest._DistTestBase.test_get_future..addrhrAr?rr?) r]r rsrrrrrrrrr~rf) rMr&r}rVrWrrrr rresexpectedrPrPrQtest_get_future)s   z-DistributedTest._DistTestBase.test_get_futurec Cs|\}}}ttt}t||}tddD]p\}}|j||||d|j||||dd|j||t d||ddd|D}|j||t d||dq4dS) N)FT)rrrrT)rrrrrr )rrrrrcSsg|]}tdt|qSr r rrPrPrQrhszNDistributedTest._DistTestBase.test_DistributedDataParallel..) r]rrrrsrYrLproductrrJr) rMrVrWrrrZuse_bucket_viewrZ gpus_listrPrPrQrvAsF    z:DistributedTest._DistTestBase.test_DistributedDataParallelc Csztdtt}tjj|dd}t }t j j ||j g|d}ttdd}ttdd}t }|D]"} || dk || jdkqtdD]} |t||} || |} W5QRX|| |D]B} | jr|| jdk || j|| jq|||td| |t td}q|S) NzQ?rrrrArirr)!rJrrrrr rrrrrErrrrKrrrrrrrrZscalerrrisnananyisinfrupdater) rM grad_is_viewr optimizerZscalerrrrloss_fnrrr rrPrPrQ&_test_DistributedDataParallel_with_ampqs>     zDDistributedTest._DistTestBase._test_DistributedDataParallel_with_ampcCsRtj|j|jdd}|jdd}t||D]\}}|||q8dS)NFr5T)rJr rrr8rhrrf)rMZddp_model_grad_not_viewZddp_model_grad_is_viewrrrPrPrQ6test_DistributedDataParallel_with_amp_and_grad_is_views zTDistributedTest._DistTestBase.test_DistributedDataParallel_with_amp_and_grad_is_viewcCs|rtnt}t|} | |dtjt|} | |dtjj | |d} t L} t j dkrt| | | dt| } nt| | jt| j} W5QRXt|d} t|d} t}|| | | |d| |d||||d|t|r dnd |dS)Nrrr9rAriTrn)BN_NETBN_NET_NO_AFFINErrr rE SyncBatchNormconvert_sync_batchnormrrrrrrrJrrrrxrKrrrrrK)rMrrrrrrrrrrrrrrrPrPrQ+_test_DistributedDataParallel_SyncBatchNormsD            zIDistributedTest._DistTestBase._test_DistributedDataParallel_SyncBatchNormcCs.d}tjjjtt|jg|d}|}tj j | |d}tjjjtt|jg|d}|}| |||} t tdd} t tdd} t} tdD]b} |||| | | || || || | | t| | D]\}}||j|jqq||j|jdS)Nr/r0rrArir)rJrErrrrrr rrrr_create_post_localSGD_optimizerrKrrrr_perform_a_train_steprrhrfr3r)rMcreate_averagerr5 learning_raterrrnet_using_post_localSGD_opt averager2post_localSGD_optrrr7rp1p2rPrPrQ$_test_post_localSGD_optimizer_paritysF   zBDistributedTest._DistTestBase._test_post_localSGD_optimizer_paritycCstjdddS)Nrir<r)rrrLrPrPrQ_create_periodic_model_averager%sz=DistributedTest._DistTestBase._create_periodic_model_averagercCstjtjj||d|dS)Nr)rr)post_localSGD_optimizerZPostLocalSGDOptimizerrJrrr)rMrrCrrPrPrQr@(sz=DistributedTest._DistTestBase._create_post_localSGD_optimizercCs.|||}|||}||dSr\)rrr)rMr6rr7rrr rrPrPrQrA.s  z3DistributedTest._DistTestBase._perform_a_train_stepc Csjd}tjjjtt|jgd}|}| |||}|}| |||}t t dd} t t dd} t } tdD]} |||| | | q|jdkrtd|i|t ddd|ji} tj|| d }||d||jd||j|j|dd =|d |d|jtd d ||dW5QRX||jddS) Nr/rrArirrZoptimizer_state_dictr map_locationrz]Loaded state dict does not contain a step counter for an averager. Setting step counter to 0.)Zexpected_warningZexpected_regex)rJrErrrrrr rr@rKrrrrrArrrrload_state_dictrrrfruZassertWarnsRegex UserWarning)rMrB chkpt_filerCrDrrFrEZdummy_post_localSGD_optrrr7rrM checkpointrPrPrQ)_test_post_localSGD_optimizer_step_reload5sX    zGDistributedTest._DistTestBase._test_post_localSGD_optimizer_step_reloadcCs"tj|j|j|jdddSNFr9rJr rrrIrJrLrPrPrQ#test_post_localSGD_optimizer_parityts zADistributedTest._DistTestBase.test_post_localSGD_optimizer_paritycCs"tj|j|j|jdddSNTr9rTrLrPrPrQ0test_post_localSGD_optimizer_parity_grad_is_views zNDistributedTest._DistTestBase.test_post_localSGD_optimizer_parity_grad_is_viewcCs"tddtfg}tj|ddS)N)rArArir)rrrrr)rMrrPrPrQ#_create_hierarchical_model_averagers zADistributedTest._DistTestBase._create_hierarchical_model_averagercCs"tj|j|j|jdddSrSrJr rrrIrXrLrPrPrQ9test_post_localSGD_optimizer_parity_with_hierarchical_sgds zWDistributedTest._DistTestBase.test_post_localSGD_optimizer_parity_with_hierarchical_sgdcCs"tj|j|j|jdddSrVrYrLrPrPrQFtest_post_localSGD_optimizer_parity_with_hierarchical_sgd_grad_is_views zdDistributedTest._DistTestBase.test_post_localSGD_optimizer_parity_with_hierarchical_sgd_grad_is_viewc Cs2tj|jt}||j|W5QRXdSr\)rJr rrrrRrJ)rMrrPrPrQ(test_post_localSGD_optimizer_step_reloads zFDistributedTest._DistTestBase.test_post_localSGD_optimizer_step_reloadcCs|\}}}t}d}t|d}t|d}t}t||} tj j | |gd} t j } t j |dddt jd|j| d} t j |dddt jd|j| d} t}|j| | | | ||||d|t| d |dS)NrArrirrT)r]rrr ONLY_SBN_NETrrr rErrrJZ channels_lastrKrkrrrrK)rMrVrWrrr bs_offsetrrrrrZ input_gpuZ target_gpurrPrPrQ8test_DistributedDataParallel_SyncBatchNorm_Channels_LastsP  zVDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_Channels_Lastc Cs|\}}}t}|g}d}t|d}t|d}|j|||||d|j|||||tdddd|D}|j|||||tdddS)NrArrrrrr )rrrrrrcSsg|]}tdt|qSr r rrPrPrQr sz\DistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm..)r]rrrr?rJr rMrVrWrrrrr^rrPrPrQ*test_DistributedDataParallel_SyncBatchNorms<    zHDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNormc CsR|\}}}t}|g}d}t|d}t|d}|j|||||dddS)NrAF)rrrrrr)r]rrrr?rarPrPrQ4test_DistributedDataParallel_SyncBatchNorm_No_Affines  zRDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_No_Affinec Cs|\}}}|g}td}t|}||dtjt|}||dtjj ||d}t |d}t |} t | d} t | d} t} t jjd<|||| |d| |d| ||| d |W5QRXdS)NrArrFT)r]rErrrr r=r>rrr~rrrJrKrrrrrrK rMrVrWrrrrrrrrrrrPrPrQ3test_DistributedDataParallel_SyncBatchNorm_2D_Input/s4         zQDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_2D_Inputc Cs|\}}}|g}td}t|}||dtjt|}||dtjj ||d}d}t } t | d} t | d} t} t jjd<|||| |d| |d| ||| d |W5QRXdS)NrArrr?FT)r]rErrrr r=r>rrrrrJrKrrrrrrKrdrPrPrQCtest_DistributedDataParallel_SyncBatchNorm_Single_Input_Per_Process]s4       zaDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_Single_Input_Per_Processc Cs|\}}}tjjt||gd}g}ttD]\}t j t ddd|dd|dt ddd|dd|dgdd}| |q6t j dd |Ddd|}td D]"}||||} | q|jj|jj} } t j| | dt j| |ddS) NrrAr?r<r'rircSs(g|] }|dddtjdqS)r?rrAr)Zpermute contiguousviewr]Z num_featuresr{rTrPrPrQrsz{DistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_Diff_Input_Sizes_Running_Value..r)r]rErrr]r rrrrJrKrrmeanrr running_mean running_vartestingZ assert_closerp) rMrVrWrrrrZinput_var_rankZ all_input_varrrkrlrPrPrQItest_DistributedDataParallel_SyncBatchNorm_Diff_Input_Sizes_Running_Values<    zgDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_Diff_Input_Sizes_Running_Valuec Csh|\}}}|g}t}t}|d}t|d|d}t|d|d} |j|||| |ddS)NrArhr`)r]r;rrrr?) rMrVrWrrrrrr^rrPrPrQDtest_DistributedDataParallel_SyncBatchNorm_Diff_Input_Sizes_gradientszbDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_Diff_Input_Sizes_gradientc Cst}tt}|r0tjj|||gd}n tj|}d}| |\}}}}|rj||}||}| dt dD] } ||} | ||| | ||| | ||d| ||} | dkr`| dks| ddkr`|| dd|| dd|| d d|| d| d || d | d || d | n| dkr||| d | |t|}q||S) NrrArr?rr<Zforward_compute_timeZbackward_compute_timeZbackward_comm_timeZ"backward_compute_comm_overlap_timer)rrrrrrErrr rZ$_set_ddp_runtime_logging_sample_raterrrrrrGrfrrJr) rMis_gpurrrrrrrrrrrPrPrQ_test_ddp_logging_datasf        z4DistributedTest._DistTestBase._test_ddp_logging_datacCsdd}ttjj|\}}}|jdd}|}||dt ||dt ||dd||d d ||d d ||d d||dd||dd||dd||dt |||ddt | }d}d} t ddtdd|D}|D]"} |d7}| | | 7} q@||dd||d| ||d|||dt| ||d |d!||d"|d#||d$|d%||d&|d'|dd(krZ||d)|d*||d+|d,d-} ||d.| ||d/d||d0d||d1d||d2d||d3d||d4d||d5dd||d6d||d7t| |d8} t td9d:td;D} || d<| |d=}||d>| ||d?d||d@d||dAd||d@|dB||dA|dB|dC}|dD}|dE}|dF}|dG}||||||||||||t}||jtjj|dHdI}|}t | }||dtdJ|d|d|d|dg}||dd<dKd|D||ddLdS)MNcSs|tjkrtj|SdS)NrnrProrPrPrQ parse_envszJDistributedTest._DistTestBase.test_ddp_logging_data_cpu..parse_envFrprr module_namerrrrrbroadcast_buffersr?Zbucket_cap_bytesifind_unused_parametersrr backend_namerrcss|] }|VqdSr\rP)r{ parameterrPrPrQr2szJDistributedTest._DistTestBase.test_ddp_logging_data_cpu..cSs|jSr\r)rxrPrPrQr5rzIDistributedTest._DistTestBase.test_ddp_logging_data_cpu..rrkZtotal_parameter_size_bytesZnum_parameter_tensors bucket_sizesZ master_portrrZ master_addrr7Ztorch_distributed_debugZTORCH_DISTRIBUTED_DEBUGZcuda_visible_devicesZCUDA_VISIBLE_DEVICESrZgloo_socket_ifnameZGLOO_SOCKET_IFNAMEZgloo_device_transportZGLOO_DEVICE_TRANSPORTrAZgloo_num_threadsZnccl_socket_ifnameZnccl_blocking_waitZnccl_async_error_handlingZ nccl_debugZ nccl_nthreadsZnccl_ib_timeoutZunused_parameter_sizehas_rebuilt_bucketsZrebuilt_bucket_sizes'prev_iteration_grad_ready_order_indicescSsg|] }t|qSrPrrirPrPrQrnszKDistributedTest._DistTestBase.test_ddp_logging_data_cpu..rh, rebuilt_per_bucket_param_indices avg_forward_compute_timeavg_backward_compute_timeavg_backward_comm_time&avg_backward_compute_comm_overlap_timeforward_compute_time_startbackward_compute_time_startbackward_compute_time_endbackward_comm_time_startbackward_comm_time_endg?)rg8Acss|]}t|VqdSr\r|rirPrPrQrsz double, float)rZset_debug_levelrINFOr]rqrrfrGrrrrYrfilterZnumelZ element_sizerreversedrrrrrkrrlrErrr)rMrrrVrWrrrrZ num_paramsZ param_sizerZdefault_gloo_threadsgrad_ready_orderexpected_orderbucket_indicesfwd_host_side_timebwd_comp_start_host_side_timebwd_comp_end_host_side_timebwd_comm_start_host_side_timebwd_comm_end_host_side_timerryrPrPrQtest_ddp_logging_data_cpus               z7DistributedTest._DistTestBase.test_ddp_logging_data_cpucCsV|\}}}|jdd}|}||dt|||d||d}ttddtdD}||d ||d }||d || |d d | |dd | |d|d| |d|d|d} |d} |d} |d} |d} | | | | | | | | | | | | dS)NTrsrrr{cSsg|] }t|qSrPr|rirPrPrQrszKDistributedTest._DistTestBase.test_ddp_logging_data_gpu..rhr}r~rrr?rrrrrrrr) r]rqrrfrGrrYrrrr)rMrVrWrrrrrrrrrrrrPrPrQtest_ddp_logging_data_gpus@           z7DistributedTest._DistTestBase.test_ddp_logging_data_gpuc Cstjt}d}|t|Zd}||\}}}}t|}| ||||||||||d| W5QRXt ||dS)Nz,should be called before training loop startsrAr?) rErrrrr,rrrrrr-) rMr expected_errrrrrrrrPrPrQtest_static_graph_api_cpus  z7DistributedTest._DistTestBase.test_static_graph_api_cpucCsNd}tj|g}tj}tjt ||}|j dj j }|||dSr)rJrrTrrrrEr=r>rrZlayer1Zbn1rrf)rMZ process_idsrZ res50_modelZres50_model_syncZprocess_group_syncrPrPrQ test_SyncBatchNorm_process_groups z>DistributedTest._DistTestBase.test_SyncBatchNorm_process_groupcCsj|tjkr"|dkr"td|d|dk rP||||t|krf|||n||||||dS)Nz Reduction fn z must specify dst!)rrrrrf)rMrqr-r~Z reduction_fnr,rPrPrQ_run_reduction_tests   z1DistributedTest._DistTestBase._run_reduction_testcCstj|j|jddk}tjjtjjfD]f}t||g |j}| |tddg |j|tddg |j}| }| |||q,tjj tjj fD]:}t||g |j}| |tddg |j|qdS)NrArFT)rJr rrrrrrrqrrrrr)rMelementr~r3r-rPrPrQ test_nccl_backend_bool_allreduces&z>DistributedTest._DistTestBase.test_nccl_backend_bool_allreducecstjjddgddgd}t|jdj}|}fddtt D}t || t |t t |D].\}}t||dj} ||q ||dS)NTFrr?rAcs"g|]}tddgjqSF)rJrqrrrrLrPrQr#szRDistributedTest._DistTestBase.test_nccl_backend_bool_allgather..)rJr rrrqrrrrrrirfr~rM)rMrr3Zinput_tensor_copy tensor_listrrr+rPrLrQ test_nccl_backend_bool_allgathers   z>DistributedTest._DistTestBase.test_nccl_backend_bool_allgathercCs(tj|jddgddgd}tjjtjjfD]}t||jd |j}tddg |j}|j |||tj ddtddg |j}| }|j |||tj ddq0tjj tjjfD]`}t||jd |j}|jdkrtddg |jn| }|j |||tj ddqdS)NTFrrArr)rJr rrrrrrrqrrrrrr)rMrr~r3r+r-rPrPrQtest_nccl_backend_bool_reduce1s,z;DistributedTest._DistTestBase.test_nccl_backend_bool_reducecsdtfddtDj}tj|ddfddttD}t|||d}|ddD]} ||qrdS)Nr<cs&g|]}jdkrtdkndqS)rrF)rrandomrrLrPrQrSszRDistributedTest._DistTestBase.test_nccl_backend_bool_broadcast..rrcs,g|]$}tddtDjqS)cSsg|]}dqSrrPrrPrPrQr[sz]DistributedTest._DistTestBase.test_nccl_backend_bool_broadcast...)rJrqrrrrrMZ tensor_sizerPrQrZsr?) rJrqrrrrrrrirf)rMZ bcast_tensorrr+rqrPrrQ test_nccl_backend_bool_broadcastMs     z>DistributedTest._DistTestBase.test_nccl_backend_bool_broadcastc stdd}fddt|D}tddd}fddt|D}t|dd}|j|j}}|d krt|n|} || ||t t |} t | |fd d } | |t|d } | j| j}} |t| ||t t | } t | || |t|d } | j| j}} |t| ||t t | } t | || |dS) Nrr?cs"g|]}tdj|qSrfrJrrrrrLrPrQrnszQDistributedTest._DistTestBase.test_DistributedSampler_padding..rAcs"g|]}tdj|qSrfrrrLrPrQrrsT)datasetZ drop_lastrcsXfddtD}t|t|gjdd|D}tt |ddS)Ncs g|]}tdgjqSr)rJZ LongTensorrrrrLrPrQrszrDistributedTest._DistTestBase.test_DistributedSampler_padding..validate_global_samples..cSsg|] }|qSrPr)r{samplerPrPrQrsr?) rrrirJrqrrrfr~r|)local_num_samplesZ world_samplesrMrrPrQvalidate_global_sampless z^DistributedTest._DistTestBase.test_DistributedSampler_padding..validate_global_samples)r) rrrror8Z num_samplesZ total_sizemathceilrfrYiterr~) rMZ dataset_sizerZdataset_tiny_sizeZ dataset_tinyZ dist_samplerrZlocal_dataset_sizeZeffective_dataset_sizeZ indices_listrZdist_sampler_added_samplesZdist_sampler_added_samples_tinyrPrrQtest_DistributedSampler_paddingcsT            z=DistributedTest._DistTestBase.test_DistributedSampler_paddingc Cst}tjd}|dkr:|jdt|j}tj ||dkr\| t tj ddddddt tD}tj|||jt||d t|D]$\}}||t|}|||qdS) Nrsr?r?rhrrcSsg|]}dqSr\rPrrPrPrQrszHDistributedTest._DistTestBase._test_allgather_object..rJ)COLLECTIVES_OBJECT_TEST_LISTrrrrrrrJr rrr[rKrrrall_gather_objectr~rMrf) rMrgather_objectsr next_rankoutput_gatheredrr r+rPrPrQ_test_allgather_objects   z4DistributedTest._DistTestBase._test_allgather_objectrsOFFrr)levelscCs|Sr\)rrLrPrPrQ!test_all_gather_object_default_pgsz?DistributedTest._DistTestBase.test_all_gather_object_default_pgcCs(t}t|}tj|d}|j|dS)Nr)r)rrrrTrrMrrrrPrPrQtest_all_gather_object_subgroups  z=DistributedTest._DistTestBase.test_all_gather_object_subgroupc svt}t|}tjd}|dkrD|jdt|j}t j ||dkrf| t t jdd|dddtt|D}d}tj||jt|||kr|nd||d ||kr||d dttDn.t|D]$\}} ||t|} || | qGd d d } | fd dttD}|td.tjddttD||j|dW5QRXdS)Nrsr?r?rhrcSsg|]}dqSr\rPrrPrPrQrszEDistributedTest._DistTestBase._test_gather_object..r)Zobject_gather_listr,rVcSsg|]}dqSr\rPrrPrPrQrsc@s eZdZdS)z>DistributedTest._DistTestBase._test_gather_object..BarN)rWrXrYrPrPrPrQBarsrcsg|]}qSrPrPrrHrPrQr szCan't pickle local objectcSsg|]}dqSr\rPrrPrPrQr srJ)rrrrrrrrrrJr rrr[rKrrZ gather_objectr~rfrMrrr) rMZpgrZmy_rankrrrZgather_on_rankrr r+rrPrrQ_test_gather_objectsF   z1DistributedTest._DistTestBase._test_gather_objectcCs|Sr\)rrLrPrPrQtest_gather_objectsz0DistributedTest._DistTestBase.test_gather_objectcCs&t}t|}tj|d}||SNr)rrrrTrrrPrPrQtest_gather_object_subgroups  z9DistributedTest._DistTestBase.test_gather_object_subgroupcs\t|j}|D]@fddttD}t||D]}||qDqdS)Ncsg|]}tqSrPrJZ zeros_likerrrPrQr$szJDistributedTest._DistTestBase.validate_net_equivalence..) rYrrr]rrrrirf)rMrnet_module_statesrrqrPrrQvalidate_net_equivalences   z6DistributedTest._DistTestBase.validate_net_equivalencec s>d}|j}d}t|tj||dd}tjjj|||jgdd}tj||dd|}t ||_ t |j }|D]^fddttD}t|t|D]*\} } | |kr|| q|| qq~t|j |j|j||jd||||kr:| } t|| D]\} || q"dS) NrAr?Fr=rrcsg|]}tqSrPrrrrPrQrAszMDistributedTest._DistTestBase.test_ddp_sync_module_states..)rrbroadcast_bucket_sizerZparams_and_buffers_to_ignore)rrJrrErFrrr rrrrYrr]rrrrirMrfrrrrZparameters_to_ignorerrh) rMrrZrank_to_broadcastrrZ new_modelrrrrqZexpected_statesr+rPrrQtest_ddp_sync_module_states+sD        z9DistributedTest._DistTestBase.test_ddp_sync_module_statesc Csd}d}d}|j}tj||dd}tj|||jd|}tjjj|||jgdd}d}|jd krl|d 7}|jdd tt |D]d} || } | tj|||jd|} t | d } || | j|tjj|jdqW5QRX|jd d t |D]} || } | t}| dkr:|d8}tj|||jd||t} t | d } || | j|tjj|jdqW5QRXdS) Nrnr?rFr=rrrhrrA)Zdivide_by_initial_world_sizeT)rrErFrJrrrr rrrrrYrrfrr synchronizerr)rMrrZ grad_scalerrrrn_itersrrrrrZ effective_wsrPrPrQtest_ddp_grad_div_uneven_inputs\sR       z=DistributedTest._DistTestBase.test_ddp_grad_div_uneven_inputsc Csd}d}d}tj|jtj||dd}tj|||jd}tjjj||j|jgd}t |}|(} t |D]} || } | qrW5QRXtd} t| | } t d d | D}|||| D]}||j||j| qtd }t|| }t d d |D}||d |D]}||j|q*tjjj||j|jgdd}t dD]} || } | qh|} || } | W5QRXt| | } |t| d || djd || dj| | D]}||jqtd| } |t| d dS)Nrhr<rFr=rrrcss|] }|jVqdSr\r7r9rPrPrQrszDDistributedTest._DistTestBase._test_ddp_profiling..rjcss|] }|jVqdSr\r7r9rPrPrQrsr?TrrvrZsearch_unused_parameters)rJr rrrErFrandrrrrrrrrrrrfrr:rxrr~r8)rMr*rrrrrrZprofiler_ctx_copyr+rrZall_reduce_event_namerr<r|Zbroadcast_event_nameZbroadcast_eventsrPrPrQ_test_ddp_profilings^                z1DistributedTest._DistTestBase._test_ddp_profilingcCstjj}|j|dSr?)rJrrrrrBrPrPrQ$test_ddp_profiling_autograd_profilers zBDistributedTest._DistTestBase.test_ddp_profiling_autograd_profilercCs6tjjj}tjjj}tjj||gd}|j|ddS)N)rr@)rJrrrr5rr)rMZcpu_actZcuda_actrFrPrPrQ!test_ddp_profiling_torch_profilers  z?DistributedTest._DistTestBase.test_ddp_profiling_torch_profilerc Csd}d}d}tj||dd}tj|||jd}t|}||j}ddtt D}t | }tj j||d } t|D],} | ||} | } | | q||j} tjjj||j|jgd }tj j||t d }|Nt| D]>}|||} | } | tjj|jd|qW5QRXt||jD]\\} }\} }|||qldS) Nrhr<r/Fr=rcSsi|]}|d|dqS)rAr?rPr%rPrPrQr`szQDistributedTest._DistTestBase.test_ddp_join_model_equivalence..rr)rErFrJrrrrr rrrrr]rrrrrrrrrrrhrrerrf)rMrrrCrr local_modelZrank_to_iter_mappingZ local_itersZ local_optimrrvrrrZ ddp_optimrZ local_tensorZ dist_tensorrPrPrQtest_ddp_join_model_equivalencesP            z=DistributedTest._DistTestBase.test_ddp_join_model_equivalencec st|j}|j}|j}|j}tj|ttj j j |||gd|d}|j dk rt| |j|j td|j ||} tj| gtjd} tj| tjjd| } d} |jr| | kr|td|jd} q|td } nt} | |j|jd t| D]}||dkr$|}nt}|Lt|trJ||}n ||}| |!|tjj"|dW5QRX| d7} qW5QRXW5QRX|jr|#| | n |$| | tjj"|d|jsp|%|j&tj|j&g|jdfd d tt'D}t(|t'd}|)|ht*d d|D|+||,}|%|-dtdS)Nr?)rrrvzregistered hook r)r~rRank  exhausted all inputs1Detected at least one rank that exhausted inputs.rcsg|]}tqSrPrrZfinal_rank_tensorrPrQrtszIDistributedTest._DistTestBase._run_uneven_inputs_test..css|]}|VqdSr\r)r{rqrPrPrQr{szHDistributedTest._DistTestBase._run_uneven_inputs_test..Zjoin_uneven_inputs).rrrrrJr rrrrErrrrrprintrqcurrent_devicerrrrrrr,rrrrr^rrrrrrfrrZ_authoritative_rankrriZassertSetEqualr|rrrG)rM test_caseiteration_mappingfind_unused_paramsrrrrrrZnum_iters_tensor min_num_itersZ total_itersrrcontextrrZmax_rankrrPrrQ_run_uneven_inputs_tests                z5DistributedTest._DistTestBase._run_uneven_inputs_testc CsBGdddtjj}tj|jt}tjt ||j}||j}t ddtj }||fD]}tjj j||jgd}d}|jdkr|}|td|jd }n|d}|td }d} |R|jd d :t|D]*} ||} | ||| d 7} qW5QRXW5QRX|| |||qndS)Ncs$eZdZfddZddZZS)zbDistributedTest._DistTestBase.test_ddp_uneven_inputs_stop_iteration_sync_bn..ModelWithCommcs ttjdddd|_dS)NrArFr=rCrDrErFrrLrNrPrQrDs zkDistributedTest._DistTestBase.test_ddp_uneven_inputs_stop_iteration_sync_bn..ModelWithComm.__init__cSs||}t||Sr\)rrrrSrPrPrQrUs  zjDistributedTest._DistTestBase.test_ddp_uneven_inputs_stop_iteration_sync_bn..ModelWithComm.forwardrVrPrPrNrQ ModelWithComms rr<rArrnrrrrTrr?)rJrErr rrr;r=r>rrrKrrrrr,rrrrrrfr) rMrZmodel_bnZ comm_modelZ model_inputrrrrnrrrPrPrQ-test_ddp_uneven_inputs_stop_iteration_sync_bnsJ         zKDistributedTest._DistTestBase.test_ddp_uneven_inputs_stop_iteration_sync_bnc sTd}d}ttdddttdddttdddt}tj||dd}t}Gd d d tj}|d}|d }|j} td |t j |d | dddtd|t j ||||| dddtd|t j ||| dddtd|t j |d | d| fddtd|t j |d | d| fddg} td|t j dt j ||| dddtd|t jt jddddddt j ||| dddg} | | trtj} | td| t ddddddg} t| D]*\}}| t|j|j|j|d dqg}| D]&}|t|j|j|j|jddq| | | |d dg}d ddg}dg}td krR|d g}|D]}|D]|D]fddtd |D}|dkr|D]$} | d kr|| d 7<q|fddt|tD||qjqbqZt !| |D]J\}}|jd kr6t"d |jd!|jd"||j#||d#|jkd$qdS)%Nrr?rrn Fr=cs,eZdZfddZddZddZZS)zODistributedTest._DistTestBase.test_ddp_uneven_inputs..UnusedParamModulecs$tt|_t|_||_dSr\)rCrDrt0runused_params_rank)rMrrNrPrQrDs zXDistributedTest._DistTestBase.test_ddp_uneven_inputs..UnusedParamModule.__init__cSs|jj|jjfSr\)rrrrLrPrPrQtask_parameterssz_DistributedTest._DistTestBase.test_ddp_uneven_inputs..UnusedParamModule.task_parameterscSs$||jkr|||S||Sr\)rrr)rMrTrrPrPrQrUszWDistributedTest._DistTestBase.test_ddp_uneven_inputs..UnusedParamModule.forward)rWrXrYrDrrUrZrPrPrNrQUnusedParamModules rrZbatch_norm_netrAr)rxrrrZlarge_conv_model small_model&unjoined_rank_with_unused_params_model$joined_rank_with_unused_params_modelZsmall_model_allreduce_hook)rxrrrrrZsmall_model_power_sgd_hook)rrrrZuse_error_feedback resnet_modelrhT)rxrrrrr<csi|] }|qSrPrPr%) baseline_iterrPrQr`_szHDistributedTest._DistTestBase.test_ddp_uneven_inputs..csi|]}|qSrPrPr%rrrPrQr`kszRunning test: z' sync interval z0 with iteration mapping Zunused_params_model)r)$rErrrrFrrrrrJrrrrrrextendrrrrrrMrxrrrrrrrr4rLr-rr)rMrrZ large_modelrZbn_netrrrrrZmodels_with_hookrZmodels_with_syncrZ test_inputZthrow_on_early_term_testsZbaseline_num_itersZiteration_offsetsZnum_uneven_ranksZiteration_mappingsZnum_early_join_ranksmappingrrrPrrQrs$   &                 z4DistributedTest._DistTestBase.test_ddp_uneven_inputsc Cst|jtjjjtjdd|j|jgd}td|j}d}t }|j ddt |D]p}|j jj}|dk r|d|||}|}|tddt |D|} ||j jj| qfW5QRX|j} || j||dS)Nr?rrnF)enablecss|] }|VqdSr\rPrrPrPrQrszSDistributedTest._DistTestBase.test_ddp_uneven_input_join_disable..)rJrrrErrrFr rrrrrrrrrrrrrfrZ _join_configrrr) rMrrrrrrrvrrZ join_configrPrPrQ"test_ddp_uneven_input_join_disables,    " z@DistributedTest._DistTestBase.test_ddp_uneven_input_join_disablec sdGfdddtj}|}tjjj||j|jgd}td}|t 2| ||}| }| W5QRXW5QRXdS)NzIntentional errorcs(eZdZfddZfddZZS)zVDistributedTest._DistTestBase.test_ddp_uneven_input_exception..ExceptionModulecs$tttjddd|_dS)Nr?Tr)rCrDrErrJrrrLrNrPrQrDs z_DistributedTest._DistTestBase.test_ddp_uneven_input_exception..ExceptionModule.__init__cs tdSr\)r)rMrZ error_strrPrQrUsz^DistributedTest._DistTestBase.test_ddp_uneven_input_exception..ExceptionModule.forwardrVrPrrNrQExceptionModules rrr?) rErrJrrr rrrrrrr)rMrZexception_modulerrrvrrPrrQtest_ddp_uneven_input_exceptions   z=DistributedTest._DistTestBase.test_ddp_uneven_input_exceptioncCsdt}|jdt|j}tjd}|dkr:tj |d}|dkr`| t tj ddddt rz| t t dd|j|kr|n dd |D}|dkr|dg}|j|kr||d|dtj|d|td d ||d|d|dkrhtjt|jkrh|dg}|j|kr<||d|dtj|d|t|d ||d|d|dkrtjt|jkr|dg}|j|kr||d|dtj|d|t|d ||d|d|dg}|j|kr||d|dtj|d|d ||d|d|j|krD|||tj|d|d |||dS) Nr?rsr?rrhri cSsg|]}dqSr\rPrrPrPrQrszMDistributedTest._DistTestBase._test_broadcast_object_list..r)rrVrr)rrrrrrrrJr rrr[rKr3rrrrrfrE)rMrVrrrsrc_rankZobjectsZsingle_obj_listrPrPrQ_test_broadcast_object_lists               z9DistributedTest._DistTestBase._test_broadcast_object_listcCs|Sr\)rrLrPrPrQtest_broadcast_object_listsz8DistributedTest._DistTestBase.test_broadcast_object_listcCs&t}t|}tj|d}||Sr)rrrrTrrrPrPrQ$_test_broadcast_object_list_subgroups  zBDistributedTest._DistTestBase._test_broadcast_object_list_subgroupcs&Gfdddtj|j}tddgddgD]\}}|j|jdt j d|j|jdt j }t j }fdd Dd fd d jD}fd d jD}t jjj||t jjj|g|||d } tjdddd|| j_| t| j|j} t jdt jd||jd} tdD]} | | | | t| jj | j D]\} }|| j |j qt| jj! | j! D]\}}|"|j |j kq|D]}|#|j dkqqbt jj$|jdq0dS)Ncs&eZdZfddZddZZS)zLDistributedTest._DistTestBase._test_ddp_ignore_params_arg..TestModelcsZ||_t|tjdddd|_|jdkrDtjdddd|_ntjdddd|_dS)Nr?Fr=rr<)rrCrDrErFrrr) TestModelrOrPrQrD&s  zUDistributedTest._DistTestBase._test_ddp_ignore_params_arg..TestModel.__init__cSs||}||}|Sr\rrSrPrPrQrU1s  zTDistributedTest._DistTestBase._test_ddp_ignore_params_arg..TestModel.forwardrVrP)rrNrQr%s rFTZ ignore_bufferrnrcsg|]\}}|jkr|qSrP)r)r{rtrrrPrQrCs zMDistributedTest._DistTestBase._test_ddp_ignore_params_arg..rcsg|]\}}d|qSrrP)r{ param_namermodel_fc2_namerPrQrHscsg|]\}}d|qSrrP)r{Zbuf_namerrrPrQrLs)rrvrurr?r=rr)%rErrrLr-rkrrrIrJrrYrrg named_modulesnamed_parametersrerr+_set_params_and_buffers_to_ignore_for_modelrFrZ_build_replicated_tensor_modulerrr rrrrrhrfrrrrr)rMrr  find_unusedruZ proxy_paramsZ proxy_buffersZproxy_param_namesZproxy_buffer_namesrrrrZmaterialized_param local_paramZ synced_paramZ proxy_paramrP)rrrrQ_test_ddp_ignore_params_arg$sn           z9DistributedTest._DistTestBase._test_ddp_ignore_params_argcCs|jdd|jdddS)NFrT)rrLrPrPrQtest_ddp_ignore_params_args z8DistributedTest._DistTestBase.test_ddp_ignore_params_argc sPGfdddtjtjjj|j|jgd}tdD]}tdd}|dkr2z|| Wnt k r"}zt |}t ||tttg}ttjjkr|tn$dd g}|d |jd ||D] }|||kd |d |q|t|kW5d}~XYnX|ddq>|| q>tdS)Ncs&eZdZfddZddZZS)z`DistributedTest._DistTestBase.test_ddp_unused_params_rebuild_buckets_exception..ToyModelcs6t|tjdddd|_tjdddd|_dSrrCrDrErFnet1net2rLToyModelrOrPrQrDsziDistributedTest._DistTestBase.test_ddp_unused_params_rebuild_buckets_exception..ToyModel.__init__cSs ||Sr\)rrSrPrPrQrUszhDistributedTest._DistTestBase.test_ddp_unused_params_rebuild_buckets_exception..ToyModel.forwardrVrPr rNrQr sr rrAr?r<rr}z net2.weightdid not receive grad for rank : Expected to be in Tz'DDP unused parameters error not raised.)rErrJrrr rrrrrr,rr-!ddp_prev_reduction_unfinished_str$ddp_recommend_find_unused_params_str ddp_outputs_not_used_in_loss_strrrrrrddp_suggest_debug_mode_strrrr"ddp_find_unused_params_enabled_strr) rMrrrrNr expected_strsunreduced_paramssrPr rQ0test_ddp_unused_params_rebuild_buckets_exceptions@        zNDistributedTest._DistTestBase.test_ddp_unused_params_rebuild_buckets_exceptioncsGfdddtjtj|jtj}dD]\}tjjj t ||jgd|d}tj dd|jd}t d D]}||}|d}|qxq:dS) Ncs&eZdZfddZddZZS)zVDistributedTest._DistTestBase.test_ddp_shared_grad_acc_unused_params..ToyModelcsNt|tjdddd|_ttd|_|j|j_tdd|_ dS)Nr<rnFr=) rCrDrErFrrrJrr>rrLr rPrQrDs  z_DistributedTest._DistTestBase.test_ddp_shared_grad_acc_unused_params..ToyModel.__init__cSs||Sr\)rrrSrPrPrQrUsz^DistributedTest._DistTestBase.test_ddp_shared_grad_acc_unused_params..ToyModel.forwardrVrPr rNrQr s r rT)rrvrrr<rr)rErrJr rrrrrrrrrKrr)rMrZstaticrrrrrPr rQ&test_ddp_shared_grad_acc_unused_paramss zDDistributedTest._DistTestBase.test_ddp_shared_grad_acc_unused_paramsc stddj}dGddd}fdd}fdd}fd d }fd d }||t|t|t|t|t|iGfd ddt jj }t jj j |jjgdfdd}tddt D} || tddt D} || t|t dd} || |d} d} t | | } t | | } t| | } || t| t| | } || t| td| td| i} || t| dS)Nr<rAc@seZdZddgZddZdS)zDDistributedTest._DistTestBase.test_ddp_device..TensorWrapperr moved_to_gpucSs||_d|_dSr)rr)rMrrPrPrQrDszMDistributedTest._DistTestBase.test_ddp_device..TensorWrapper.__init__N)rWrXrY __slots__rDrPrPrPrQ TensorWrappersrcsTt|dttdd|D|djjj|d|dS)Nr?css|] }|jVqdSr\rrrPrPrQrszbDistributedTest._DistTestBase.test_ddp_device..tuple_and_list_validator..r)rr~rfr|rindexrr]) expected_lenrMrPrQtuple_and_list_validatorszODistributedTest._DistTestBase.test_ddp_device..tuple_and_list_validatorcsF|jt|jjj|jjj|jjjj|j|jSr\)rf_fieldsEXPECTED_FIELDSrGrrrHrr]rLrPrQnamedtuple_validatorszKDistributedTest._DistTestBase.test_ddp_device..namedtuple_validatorcs8|jpt|jjdk|jj|_d|_|jS)NrT)rrrrrrrr]rLrPrQcustom_type_validatorszLDistributedTest._DistTestBase.test_ddp_device..custom_type_validatorcstd|ktd|kdttdd|D|tdjjj |td|tdS)Nrr?css|] }|jVqdSr\rrrPrPrQrszXDistributedTest._DistTestBase.test_ddp_device..dict_validator..) rr!rrfr~r|r]rrrr]rLrPrQdict_validators "zEDistributedTest._DistTestBase.test_ddp_device..dict_validatorcs*eZdZfddZfddZZS)z?DistributedTest._DistTestBase.test_ddp_device..ToyModelcs ttjdddd|_dSrr_selfrNrPrQrDs zHDistributedTest._DistTestBase.test_ddp_device..ToyModel.__init__cs&t||||}||Sr\)rr^r)r&rT expected_typeZ fwd_tensorrM validatorsrPrQrUs zGDistributedTest._DistTestBase.test_ddp_device..ToyModel.forwardrVrPr(rNrQr s r rcs(tdD]}||}|qdS)Nri)rrr)rZ input_typerrvrrPrQ train_iter$s  zADistributedTest._DistTestBase.test_ddp_device..train_itercss|]}tddVqdS)r<NrJrKrrPrPrQr+sz@DistributedTest._DistTestBase.test_ddp_device..cSsg|]}tddqSrXr+rrPrPrQr0szADistributedTest._DistTestBase.test_ddp_device..rnrr?)rErFrrrrYTestNamedTupleInput_0rpdictrJrrrrrKrtyper!)rMrrrr"r#r$r r*rrrrGrHrP)rrrMr)rQtest_ddp_devices\               z-DistributedTest._DistTestBase.test_ddp_devicecsd}d}tj||jdtj||jdGfdddtjj}tjjj|jjgd}t}||t |t }||t |dS)Nrnr<rcs,eZdZfddZfddZZS)zKDistributedTest._DistTestBase.test_ddp_namedtuple..NamedTupleModulecsttdd|_dS)Nr<r?rr%rNrPrQrDUs zTDistributedTest._DistTestBase.test_ddp_namedtuple..NamedTupleModule.__init__csbt||d|dt||jt|j|j|t |j|jS)NzExpected type r) rr^r.rfr r!rGrHrrJmul)r&rr'rGrHrMrPrQrUYszSDistributedTest._DistTestBase.test_ddp_namedtuple..NamedTupleModule.forwardrVrPr1rNrQNamedTupleModuleTs r2r) rJrrrErrrr r,r.rp)rMrrr2rrrPr1rQtest_ddp_namedtupleJs   z1DistributedTest._DistTestBase.test_ddp_namedtuplec CsJd}d}t}tj|jtjjjt |j|jgdd}tj |||jd}tj |||jd}t dD]}|ddkr||}n||}| } | |j} |ddkrtj|dg|jtjd } ntj||g|jtjd } | } || | qltjjjt |j|jgd d}t dD]}|dkrH|| } | nz|| } | Wntk r,} zt| }t||d }tttd |jd |g}ttjjkr|tn$ddg}|d|jd ||D]"}| ||kd|d|q|!t"|kW5d} ~ XYnX|!ddq"t#dS)Nrr<TrrrrArrrFr?6Parameter indices which did not receive grad for rank r r} lin2.weightr rrDDP error not raised)$rrrJr rrrErrrrKrrrrreducer_get_local_used_maprqint32rfr,rr-rrrrrrrrrrrrr)rMrrrr random_input ones_inputrrvrlocal_used_mapr+variable_usage_tensorrNrunused_param_indexrrrrPrPrQ'test_ddp_control_flow_same_across_ranksns~                 zEDistributedTest._DistTestBase.test_ddp_control_flow_same_across_ranksc Cs t}tj|jtjjjt |j|jgdd}tj dd|jd}tj dd|jd}d}| t |BtdD]2}|ddkr||}n||}|}|qvW5QRXt||| t d BtdD]2}|ddkr||}n||}|}|qW5QRXt|d dS) NTrrrr<rz1Your training graph has changed in this iterationrArafExpected to have finished reduction in the prior iteration before starting a new one. This error indicates that your training graph has changed in this iteration, e.g., one parameter is used in first iteration, but then got unused in the second iteration. this is not compatible with static_graph set to True. Parameter indices which did not receive grad forz#Expected to have finished reduction)rrrJr rrrErrrrKrrr,rrrr-) rMrrr;r<rrrvrrPrPrQtest_invalid_static_graphs:        z7DistributedTest._DistTestBase.test_invalid_static_graphc slddGfdddtjt}tj|jtjjj |j|j|jgdd}tj |jd}tj |jd}t dD]}|d d kr||}n||}| }||j}|d d krtj|d g|jtjd } ntj|d g|jtjd } |} || | qtjjj |j|j|jgd d}t d D]}|d krj|| }|nz|| }|Wntk rN} zt| } t|| d } tttd|jd| g}ttjjkr|tn$ddg}|d|jd||D]"}| || kd|d| q |!t"| kW5d} ~ XYnX|!ddqDt#dS)Nrr<cs,eZdZfddZfddZZS)z\DistributedTest._DistTestBase.test_ddp_control_flow_different_across_ranks..ToyModelcs<t|tjdddd|_tjdddd|_||_dSrrCrDrErFrrrrr rPrQrDszeDistributedTest._DistTestBase.test_ddp_control_flow_different_across_ranks..ToyModel.__init__csRt|tj|jdo"|jdk}|r>|t||St||SdS)Nrr?) rJr`rrrrrrrr)rrrPrQrUs zdDistributedTest._DistTestBase.test_ddp_control_flow_different_across_ranks..ToyModel.forwardrVrPr rrrNrQr sr TrrrrArr4r?Fr5r r}r6r rrr7)$rErrrrJr rrrrrKrrrrr8r9rqr:rfr,rr-rrrrrrrrrrrrr)rMrrr;r<rrvrr=r+r>rNrr?rrrrPrDrQ,test_ddp_control_flow_different_across_rankss               zJDistributedTest._DistTestBase.test_ddp_control_flow_different_across_ranksc Csd}|j|krtn ddtD}t}|d|}d}t||kr\||||d7}q8dg}tj|||d||dt|jtt|t dtjg||dW5QRXdS)NrcSsg|]}dqSr\rPrrPrPrQrbszJDistributedTest._DistTestBase.test_scatter_object_list..r?rzMExpected argument scatter_object_output_list to be a list of size at least 1.) rrrrr~rZscatter_object_listrfrr,)rMrrrrZoutput_obj_listrPrPrQtest_scatter_object_list\s0     z6DistributedTest._DistTestBase.test_scatter_object_listc Csvtjdgtjdtjdgtjdtjdgtjdtjdgtjdtjdgtjdtjdgtjdg}dd|D}|S)NrrrcSsg|] }|qSrP)Z to_sparserrPrPrQrszeDistributedTest._DistTestBase._generate_sparse_tensors_for_bucket_assignment_test..)rJrrkrl)rMrtensors_sparserPrPrQ3_generate_sparse_tensors_for_bucket_assignment_testzs zQDistributedTest._DistTestBase._generate_sparse_tensors_for_bucket_assignment_testc Cstjtddtjjd}dtjd<tjttddd}tj |j t d}tj jj||j |j g|d }d }|t|6|}|rtj|d g|jd }nt|d g}W5QRX|rt||t|dS) Nrr@r/rrrrnrr/rrzNo support for sparse tensors.i)logger)rrTrrrrrrrJr rrrrErrrrr,rHZ"_compute_bucket_assignment_by_sizerKr-r)rM use_loggerr  group_to_userrrGrrPrPrQ'_test_compute_bucket_assignment_by_sizes:   zEDistributedTest._DistTestBase._test_compute_bucket_assignment_by_sizecCs|jdddSNFrLrNrLrPrPrQBtest_compute_bucket_assignment_by_size_sparse_error_without_loggersz`DistributedTest._DistTestBase.test_compute_bucket_assignment_by_size_sparse_error_without_loggercCs|jdddSNTrPrQrLrPrPrQ?test_compute_bucket_assignment_by_size_sparse_error_with_loggersz]DistributedTest._DistTestBase.test_compute_bucket_assignment_by_size_sparse_error_with_loggercCs|rd}|t|}||fSttjjk}|jdkrnt|tjj kr^|s^d}|t|}q~d}| t}nd}|t|}||fS)Nz'DDP expects same model across all ranksr#Caught collective operation timeoutzappears not to match) rr,rrrrrrrrr)rMrMrrr;Zis_detail_dbg_moderPrPrQ2_determine_expected_error_verify_model_across_ranks     zPDistributedTest._DistTestBase._determine_expected_error_verify_model_across_rankc Cstjtddtjjd}dtjd<tjttddd}tj |j | |\}}t d}tjjj||j |j g|d }t|j dkrd nd d |j_|B|rt|jt||jnt|jt|t|W5QRX|r|j dkrt||t|dS) Nrr@rIrrrnrJrrrr<r?)rrTrrrrrrrJr rrrVrrErrrrFrrrrrYrrKrr-)rMrLr rMr;rrrPrPrQ_test_verify_model_across_ranksB      z.ToyModelcs<t|tjdddd|_tjdddd|_||_dSrrCrr rPrQrDsz[DistributedTest._DistTestBase._test_different_graph_across_ranks..ToyModel.__init__cSs4|jdkr |t||St||SdSr)rrrrrrSrPrPrQrUs zZDistributedTest._DistTestBase._test_different_graph_across_ranks..ToyModel.forwardrVrPr rNrQr sr r.T)rrvrrrr<r)rErrJrrrr rrrrrKrrr) rMrvrrrrr;rrvrrPr rQ"_test_different_graph_across_rankss$   z@DistributedTest._DistTestBase._test_different_graph_across_rankscCsp|jdd}||dd|jdd}||ddt||D]\}}|||qVdS)NT)rvrzrr)rkrrrGrrhrrf)rMZ base_modelZ static_modelrrrPrPrQ!test_different_graph_across_rankss z?DistributedTest._DistTestBase.test_different_graph_across_rankszUMacOS uses uv transport which does not have as robust error handling as tcp transportc Cstd|jg}tdD]}tt|qtdd}tj|dtdD]}tt|qNtj|ddd}d}|j|kr| t d |d tj|dW5QRXn@|j|krd |jd |}| t |tj|dW5QRX|j d ddS) Nr<rAr@rTr/wait_all_ranksr?rr failed to pass monitoredBarrierb successfully reached monitoredBarrier, but received errors while waiting for send/recv from rank ) rJrrrrrrKrmonitored_barrierrr,rK)rMrrr/ failed_rankr err_regexrPrPrQtest_monitored_barrier_gloo"s,       z9DistributedTest._DistTestBase.test_monitored_barrier_glooc Csnd}d}tjddgd}|j|kr&dS|jdkr^|td|dt||W5QRXn t||dS)Nr?r'rrrro)rrTrrr,rr)rMrsr/rrPrPrQ$test_monitored_barrier_gloo_subgroupKs   zBDistributedTest._DistTestBase.test_monitored_barrier_gloo_subgroupc Csbtjtddtt|jDtddtjjd}tjtddtt|jDtjj d}t j d|j d |j g}| |td d|j d krttjjkrd }nd }|t|| |tddW5QRXnt|rdddtdt|jD}d|d}nd}d|d}tdd}|t||j||dW5QRX|jdddS)Ncss|] }|VqdSr\rPrrPrPrQreszWDistributedTest._DistTestBase._test_monitored_barrier_allreduce_hang..rr@)rr/rcss|] }|VqdSr\rPrrPrPrQrlsrr<rrnrzTimed out waitingrUr'r}cSsg|] }t|qSrPr|rrPrPrQrszXDistributedTest._DistTestBase._test_monitored_barrier_allreduce_hang..r?Ranks rorrnrqr)rrTrYrrrrrrrrJrrrrrrrrr,rrrrK) rMrnZnccl_pgZgloo_pgrrtrank_strZexpected_first_fail_rankZ!monitored_barrier_timeout_secondsrPrPrQ&_test_monitored_barrier_allreduce_hangbs> "  zDDistributedTest._DistTestBase._test_monitored_barrier_allreduce_hangcCs|jdddS)NFryr{rLrPrPrQ%test_monitored_barrier_allreduce_hangszCDistributedTest._DistTestBase.test_monitored_barrier_allreduce_hangcCs|jdddS)NTryr|rLrPrPrQ4test_monitored_barrier_allreduce_hang_wait_all_ranksszRDistributedTest._DistTestBase.test_monitored_barrier_allreduce_hang_wait_all_ranksc Cshtjtddtt|jDd}tdd}|jdkrd|t d|jd| |W5QRXdS)Ncss|] }|VqdSr\rPrrPrPrQrsz[DistributedTest._DistTestBase.test_monitored_barrier_gloo_rank_0_timeout..rvrr@rz timed out in monitoredBarrier) rrTrYrrrrrrr,rr)rMrr/rPrPrQ*test_monitored_barrier_gloo_rank_0_timeouts   zHDistributedTest._DistTestBase.test_monitored_barrier_gloo_rank_0_timeoutc Csd}tdd}d}|j|krH|td|tj|dW5QRXn@|jdkrd|jd|}|t|tj|dW5QRXdS)NrAr@rrrr?rp)rrrr,rrr)rMZexpected_first_failed_rankr/rrtrPrPrQ$test_monitored_barrier_failure_orders    zBDistributedTest._DistTestBase.test_monitored_barrier_failure_orderc Csj|jdkrftdd}dddtdt|jD}d|d }|t|tj |d d W5QRXdS) Nrr'r@r}cSsg|] }t|qSrPr|rrPrPrQrszWDistributedTest._DistTestBase.test_monitored_barrier_wait_all_ranks..r?rxroTrm) rrrrrrrr,rrr)rMr/rzrtrPrPrQ%test_monitored_barrier_wait_all_rankss     zCDistributedTest._DistTestBase.test_monitored_barrier_wait_all_ranksc Cst}tjjj||j|jgd}ddd}|\}}||}| ||t}dg}tjjj ||tjjj||j|jgd}ddi}|\}}||}| ||t}tjjj||j|jgd}|\}}|jdkrt t |d| tjtdtjtdg|td||W5QRX|dd}|td ||W5QRX| tjtdtjtdgdS) Nrza.weightzb.weightrrr?zExpected param to name mappingrzParam with name)rrJrErrr r_build_params_for_reducer"_build_debug_param_to_name_mappingZassertDictEqualrrr.rrrrr)rMrrexpected_mapping net_paramsrparam_to_name_mappingZparams_to_ignorerPrPrQ*test_ddp_build_debug_param_to_name_mappings\              zHDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mappingcCsdGdddtj}|}tjjj||j|jgd}ddi}|\}}||}| ||dS)Ncs$eZdZfddZddZZS)zcDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mapping_requires_grad..Netcs*ttdd|_|jjddS)Nr<F)rCrDrErFrr>rrLrNrPrQrD( s zlDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mapping_requires_grad..Net.__init__cSs ||Sr\rrSrPrPrQrU/ szkDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mapping_requires_grad..Net.forwardrVrPrPrNrQr' s rrrz lin.weight) rErrJrrr rrrrf)rMrrrrrrrrPrPrQ8test_ddp_build_debug_param_to_name_mapping_requires_grad s    zVDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mapping_requires_gradc sttjjk}GdddtjGfdddtj}|}g}|r|D]D\}}||jjj krN|j ddD] \}} |d|} | | qpqNt jj j|||jjj|jjjg} nt|jj|jjjg} g} g} i}d}|D]~\}}|j ddD]h\}} |d|} ||| <| |kr4|d 7}|| krJ| | n |r`||jjj kr| | qqt jj j||j|jgd }d \}}t ||}td D]}|dkr||}|}|nz||}|}|Wntk r}zt|}||d d}| D]B}|||kp6||t|||kd||d|q"| D]}|||kqj|D]}|||kqW5d}~XYnX|ddqdS)Ncs$eZdZfddZddZZS)z^DistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error..SubModulecs:ttd|_t|_t|_tj dddd|_ dS)Nrrir<Fr=) rCrDr embedding_netrrrrrErF lin_layerrLrNrPrQrDA s   zgDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error..SubModule.__init__cSs$||}||}|j|}|Sr\)rrrrGrSrPrPrQrUH s   zfDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error..SubModule.forwardrVrPrPrNrQ SubModule@ s rcs&eZdZfddZddZZS)z\DistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error..MyModelcst|_dSr\)rCrD sub_modulerL)rrOrPrQrDQ s zeDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error..MyModel.__init__cSs ||Sr\)rrSrPrPrQrUU szdDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error..MyModel.forwardrVrPrrNrQMyModelP srF)Zrecurserrr?r)r<rArAzdid not receive gradzDid not find index z for zExpected error was not raised!)rrrrrErrrrrrrrJrrrrrHrYmodulesr rrrrrr,rfindrr)rM ignore_sparseZdebug_mode_offrrZsparse_embedding_fqnsrtrZparameter_namerZfqnZunused_modulesZexpected_unused_param_fqnsZused_param_fqnsZfqn_to_param_indexrrrrrrrvrrNZunused_param_substrZunused_param_fqnZused_param_fqnZsparse_param_fqnrPrrQ-_test_ddp_multiple_nested_unused_params_error= s            $zKDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_errorcCs|jdddS)NFrrrLrPrPrQ,test_ddp_multiple_nested_unused_params_error szJDistributedTest._DistTestBase.test_ddp_multiple_nested_unused_params_errorcCs|jdddS)NTrrrLrPrPrQ8test_ddp_multiple_nested_unused_params_err_ignore_params szVDistributedTest._DistTestBase.test_ddp_multiple_nested_unused_params_err_ignore_paramscCs|j}tj|t}t|}tjjj ||gd}tj dddd}t|}tjjj ||gd}tj dd|d}tj dddd|d}|||f|||fg}|D]J} | \} } } |jd kr| | t d D]} || | | | qq|jd d dS) NrrArFrZtrack_running_statsr<rrirrrqr)rrJr rrrrrErrr=rKevalrrfrK)rMrrrZ syncbn_modelZlocal_syncbn_modelrZ inp_syncbnrtestZ test_modelZtest_local_modelZtest_inprrPrPrQtest_ddp_inference sD        z0DistributedTest._DistTestBase.test_ddp_inferencec CsX|j}tj|tjdddd|}tjjj||gd}tjj B}t dD]2}t dddd|}||}| }|qRW5QRXtd krtd |}n td |}|g||j} |jd krT| tjj B}t dD]2}t dddd|}| |}| }|qW5QRXtd kr>td |}n td |}|g|dS) NrArFrrrr<rir?Z_all_gather_baserir)rrJr rrEr=rrrrrrrKrrrsrrrrrf) rMrrr+rrrvrZall_gather_callsZmodel_inferencerPrPrQ!test_ddp_sync_bn_training_vs_eval s:          z?DistributedTest._DistTestBase.test_ddp_sync_bn_training_vs_evalc CsZt|j}tjjj||jgd}d}|t|| iiW5QRXt ||dS)Nrzmust be callable) rr rrJrErrr TypeErrorrr-)rMrrrPrPrQtest_ddp_python_error_logged!s z:DistributedTest._DistTestBase.test_ddp_python_error_loggedcs4|j}tj|Gdddtjj}fdd||}t|}tjjj ||gd}tjjj ||gdd}t dd }t t t d }|D]}td D]}|||d } | } | |||||d } |t| ||| } | ||t||D]\} }|| |qqqdS) Ncs$eZdZfddZddZZS)z\DistributedTest._DistTestBase.test_ddp_static_graph_nested_types..NestedOutputModulecs ttjdddd|_dS)Nrr?Fr=rrLrNrPrQrD>!s zeDistributedTest._DistTestBase.test_ddp_static_graph_nested_types..NestedOutputModule.__init__cSsr|dkr&||||||ffS|dkrL||||||ggS|dkrn||d||idSdS)NrrYr-rror)rMr output_typerPrPrQrUB!s$zdDistributedTest._DistTestBase.test_ddp_static_graph_nested_types..NestedOutputModule.forwardrVrPrPrNrQNestedOutputModule=!s rcsd}t|tjr|St|tr>|D]}||7}q*n>t|tsRt|trj|D]}||7}qVntdt ||S)NrjzUnknown model output type ) r^rJr_rr-r]rrYrr.)Z model_outputrrarTget_lossrPrQr[!s   zRDistributedTest._DistTestBase.test_ddp_static_graph_nested_types..get_lossrTrAr<r)rYrr-r)r)rrJr rrErrrrrrKrYrr-rrrrrr^rhrrf)rMrrrZmodel_static_graphrZ type_mappingrrrvrZ out_staticZ loss_staticrZp_staticrPrrQ"test_ddp_static_graph_nested_types2!sH            z@DistributedTest._DistTestBase.test_ddp_static_graph_nested_typesc Cstj|jGdddtj}||j}tjdd|jd}t ddgddgD]b\}}t ||jg|j||d}t d D]8}||}| |d j |d |d} | q|qTdS) Ncs$eZdZfddZddZZS)zSDistributedTest._DistTestBase.test_ddp_returns_tensor_with_no_grad..MyModelcs2ttjdddd|_tjdddd|_dSr)rCrDrErFrrrLrNrPrQrD!s z\DistributedTest._DistTestBase.test_ddp_returns_tensor_with_no_grad..MyModel.__init__cSs8|t||}|}|}|jr0t||fSr\)rrrrrdetachrr)rMrTrrPrPrQrU!s  z[DistributedTest._DistTestBase.test_ddp_returns_tensor_with_no_grad..MyModel.forwardrVrPrPrNrQr!s rr?r<rTF)rrrvrrr)rJr rrrErrrKrLr-rrrrrr) rMrrrrrrrrvorPrPrQ$test_ddp_returns_tensor_with_no_grad!s"  zBDistributedTest._DistTestBase.test_ddp_returns_tensor_with_no_gradc s Gfdddtjtj|j}dD]h}tjjj||jg|d}tjdddd}t d D]2}|||d d }| }| | |j qfq2tjjj||jgd d}tjdddd}t d D]*}||d |d dkd }| }| q||j dS)Ncs&eZdZfddZddZZS)zRDistributedTest._DistTestBase.test_detect_ddp_is_actually_static..ToyModelcs2t|tjdddd|_tdd|_dSrrrLr rPrQrD!sz[DistributedTest._DistTestBase.test_detect_ddp_is_actually_static..ToyModel.__init__cSs8|r$|r|||S||Sn|||SdSr\)rr)rMrTrdynamicrPrPrQrU!s  zZDistributedTest._DistTestBase.test_detect_ddp_is_actually_static..ToyModel.forwardrVrPr rNrQr !sr rrr?r<r rrF)rrTrAr)rErrJr rrrrrKrrrrr8Z_ddp_graph_staticr) rMrrrrrrvrrrPr rQ"test_detect_ddp_is_actually_static!s4    z@DistributedTest._DistTestBase.test_detect_ddp_is_actually_staticc CsGdddtj}||j}dD]}t||jg|jd||d}ddtdD}td D]}|tjd d |jd }|||d |d |d d\} |d <|d <|d <tt |D]<}t ||r| ||j dq| ||dj dq| q\q$dS)Ncs,eZdZfddZddZddZZS)zJDistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd..MyModelcs>ttjdddd|_tjdddd|_|jjj|_dSr)rCrDrErFrrrrrLrNrPrQrD!s zSDistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd..MyModel.__init__cSstjdd|jd}|S)Nr?r<r)rJrKr)rMrrPrPrQZ __init_opt!szUDistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd..MyModel.__init_optcSsdt||}||}|dkr*|}|dkr:|}|dksLt|sT|}|||d|ifS)Nrq)rrrr_MyModel__init_optrJ is_tensor)rMrTopt_1opt_2 opt_nestedrPrPrQrU!s zRDistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd..MyModel.forward)rWrXrYrDrrUrZrPrPrNrQr!s rrF)rrrurvrcSsg|]}dqSr\rPrrPrPrQr"szMDistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd..rhrAr?r<rr)rrrrq)rErrrrrrrJrKr~rrfZgrad_fnrjr) rMrrrrrrrrTrvrPrPrQ_test_ddp_new_tensor_in_fwd!s4  z9DistributedTest._DistTestBase._test_ddp_new_tensor_in_fwdcCs |jddS)NFrrrLrPrPrQtest_ddp_new_tensor_in_fwd"sz8DistributedTest._DistTestBase.test_ddp_new_tensor_in_fwdcCs |jddS)NTrrrLrPrPrQ'test_ddp_new_tensor_in_fwd_static_graph"szEDistributedTest._DistTestBase.test_ddp_new_tensor_in_fwd_static_graphcs|j}tj|t|tj|fdd}tjjjjj }tjjjjj }||fD],}t |}tjjj ||jgd}| |||tjjj t||jgdd} tjdd|d} tdD]} || } ||krt| j} | D]}t|q| | }||kr8t| j} | D]}t|q&tjsT||| | |r||kr||| qtqXdS) Ncs@dd|D}fdd|D}r,|Stj|dS)NcSsg|] \}}|qSrPrPr{rr@rPrPrQr-"szkDistributedTest._DistTestBase._test_ddp_buffer_hook_allreduce..buffer_comm_hook..cs"g|]}tj|jddqS)T)rVr)rrrr)r{r@rrPrQr0"s)rerJr#Z collect_allr)rrergZfutsreturn_futuresrrQbuffer_comm_hook,"s zWDistributedTest._DistTestBase._test_ddp_buffer_hook_allreduce..buffer_comm_hookrF)rrurAr<r)rrJr rrrErrZ_BufferCommHookLocationZ PRE_FORWARDZ POST_FORWARDr:r_register_buffer_comm_hookrrrKrrrYrrgrrrrmrr)rMrrrZ hook_pre_fwdZ hook_post_fwdZhook_run_locationr model_ddpmodel_ddp_no_hookrrrZmodel_no_hook_buffersrq loss_no_hookrPrrQ_test_ddp_buffer_hook_allreduce&"sZ              z=DistributedTest._DistTestBase._test_ddp_buffer_hook_allreducecCs|jdddS)NTrrrLrPrPrQ,test_ddp_buffer_hook_allreduce_return_futureq"szJDistributedTest._DistTestBase.test_ddp_buffer_hook_allreduce_return_futurecCs|jdddS)NFrrrLrPrPrQtest_ddp_buffer_hook_allreduce{"sz.buffer_comm_hook..)reZ_default_broadcast_coalesced)rrergrPrPrQr"szZDistributedTest._DistTestBase.test_ddp_broadcast_buffer_via_hook..buffer_comm_hookrrAr<r)rrJr rrr:rErrrrrrKrrrmr) rMrrrrrrrrrrPrPrQ"test_ddp_broadcast_buffer_via_hook"s2        z@DistributedTest._DistTestBase.test_ddp_broadcast_buffer_via_hookc s|j}tj|t|tj|Gdddtj}||}tjjj||jgdtj dd|d}t dD]}|dkrj j dj _ | }|fd d t tD}t|j j |d}|ddD]} ||| qqtdS) Ncs$eZdZfddZddZZS)zODistributedTest._DistTestBase.test_ddp_broadcast_buffer..NetWithBufferscsFttjdddd|_tjdddd|_|dtdddSr;rBrLrNrPrQrD"s zXDistributedTest._DistTestBase.test_ddp_broadcast_buffer..NetWithBuffers.__init__cSs|||Sr\)rHrGrSrPrPrQrU"szWDistributedTest._DistTestBase.test_ddp_broadcast_buffer..NetWithBuffers.forwardrVrPrPrNrQr:"s r:rrAr<rrr?csg|]}tjjqSrP)rJrbrr@rrrPrQr"szKDistributedTest._DistTestBase.test_ddp_broadcast_buffer..)rrJr rrrErrrrKrrr@rrrrrirf) rMrr:rrrrZbufsZ rank_0_bufrlrPrrQtest_ddp_broadcast_buffer"s*      z7DistributedTest._DistTestBase.test_ddp_broadcast_bufferz8Only Nccl & Gloo backend support DistributedDataParallelcCst}|j}||}tjjjt||jgd}| }| dd}| |tj |}tjjj||jgd}| }| dd}||dS)NrZ has_sync_bnTF)r;rr rJrErrrrrrGrr=r>r)rMrrrZ no_sync_bnrZsync_bn_loggedrrPrPrQtest_sync_bn_logged"s$     z1DistributedTest._DistTestBase.test_sync_bn_loggedc CsnGdddtjj}|j}||}ddlm}|dtjjj||gd}W5QRXt d|}tj dgg|d d }tj d g|d d }tj d g|d }|||d } |j j j } |j j} t|| |} ||| |j j j } |j j}|| | ||| | ||j||j||j||j j j j||j j jj||j jjdS)Ncs$eZdZfddZddZZS)zMDistributedTest._DistTestBase.test_stateless_api_with_ddp..MockModulecs4ttjdd|_td}|d|dS)Nr?r@)rCrDrJrErFr4rrI)rMr@rNrPrQrD"s  zVDistributedTest._DistTestBase.test_stateless_api_with_ddp..MockModule.__init__cSs|||jSr\)r4r@rSrPrPrQrU"szUDistributedTest._DistTestBase.test_stateless_api_with_ddp..MockModule.forwardrVrPrPrNrQ MockModule"s rr)_ddp_replicated_tensorFr)r?r?rT)rrrjr)zmodule.l1.weightzmodule.l1.biasz module.buffer)rJrErrrrCrrrrrqrr4rrr@ _statelessZfunctional_callrfrZassertIsNotNonerrr>)rMrrrrrTrr>r@rZ prev_weightZ prev_bufferr*Z cur_weightZ cur_bufferrPrPrQtest_stateless_api_with_ddp"s@            z9DistributedTest._DistTestBase.test_stateless_api_with_ddpc s$Gfdddtjdd}dd}dd}}}|j||j||j||j||j||j|t||j|jgd }t d d }||}|||j}| ||| | d d |D} | | d|jjj| | d|jjjdS)Ncs&eZdZfddZddZZS)zTDistributedTest._DistTestBase.test_ddp_forward_backward_hook..DummyTestModelcs*t|tdtdd|_dS)NrrA)rCrDrJrrErFrrL)DummyTestModelrOrPrQrD,#s z]DistributedTest._DistTestBase.test_ddp_forward_backward_hook..DummyTestModel.__init__cSs ||Sr\rrSrPrPrQrU1#sz\DistributedTest._DistTestBase.test_ddp_forward_backward_hook..DummyTestModel.forwardrVrPrrNrQr+#srcSstj|dSr)rE functionalr)rrrPrPrQ relu_hook4#szODistributedTest._DistTestBase.test_ddp_forward_backward_hook..relu_hookcSs tj|Sr\)rErZgelur_inputr rPrPrQ gelu_hook7#szODistributedTest._DistTestBase.test_ddp_forward_backward_hook..gelu_hookcSstj|dfSr)rErZcelurrPrPrQ celu_hook:#szODistributedTest._DistTestBase.test_ddp_forward_backward_hook..celu_hookrrnrAcSsg|] }|jqSrP)r)r{rrPrPrQrN#szPDistributedTest._DistTestBase.test_ddp_forward_backward_hook..rr?)rErrZregister_forward_pre_hookZregister_forward_hookZregister_backward_hookrrrrJrrfrrrrrr>) rMrrrrrZ input_dataZ output_localZ output_ddpZ ddp_gradsrPrrQtest_ddp_forward_backward_hook'#s2            zrArDrGrPZskip_collectiverQrRrSrTrUrVrWr`rarbrcrhrqrrrtrurvryrrrrrrrrrrrrrrrrrrrrrrmrJrkrrrrrrrrrrrrrrrrrrrrrr staticmethodrrrrrrrrrrrrrrrrrrrr r rrrrrrrrrrr r"r#r$r%r&r'r(r)r*r.r0r6r?r@rArBrCrDrHrLrNrPrQrRrSrTrVrWrXrYr"rZr[r\r]r^r_r`rarbrcrdrerfrgrhrkrlrmrnrorprqrsrtrwrxryr{r|rrrrrrrrrrrrrrrrrrrrrr5rrrrrrrr4rrr rr&rrr$r)r,rvr8r:r?rIrJr@rArRrUrWrXr$rZr[r\r_rbrcrerfrnrorqrrrskipIfNoTorchVisionrrrrrrrrrrr(r,rrrrrrrrrrrrrrrrrrrrrrrrr/r3r@rBrErFrHrNrRrTrVrWrYrZr[r^r_rerhrirjrkrlrurwr{r+r}r~rrrrrrrrrrrrrrrrrrrrrrrrrrrrPrPrPrQ _DistTestBaseUs@  !                                $    #   "  E               ,    '  M     "  -        H         !                  )   (     2 @ 8                              3      !         !    " '     $!          ")    H<#  ! ; 9     3Q      +7 ;/?      + (  ) ) & E  & N    /    , 4<     .k 6 @  O    [    2  "  f  !  P  .  e '    6        \   "  #5       < t     & ,  S ! -3  K   % " 2  (s rN)rWrXrYrrPrPrPrQrRTsrR)rrLrrrrrr* collectionsrr contextlibrrdatetimer functoolsrtypingrr r r ZnumpyrrJZ torch.cudaZtorch.distributedrrZ6torch.distributed.algorithms.model_averaging.averagersZ algorithmsZmodel_averagingrZHtorch.distributed.algorithms.model_averaging.hierarchical_model_averagerZhierarchical_model_averagerrZ2torch.distributed.algorithms.model_averaging.utilsutilsrZtorch.nnrEZtorch.nn.functionalrrZtorch.nn.utils._statelessrZtorch._utils_internalr r7r rrZtorch.cuda.amprrZ+torch.distributed.algorithms.ddp_comm_hooksrrrrrrrrrrrrrZtorch.distributed.utilsrrZtorch.nn.parallelrZtorch.nn.parallel.distributedrZ*torch.testing._internal.common_distributedrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.Z$torch.testing._internal.common_utilsr/r0r1r2r3r4r5r6r7Z/torch.distributed.optim.post_localSGD_optimizerrrKZtorch.utils.data.distributedr8rr ImportErrorrrrrr:r[r1barrKZfoo_cpu_tensorrrrrrrrrr(r!r,rprrrsgetenvrtrrrrrrrrrrrrrrrrrrrrr;r<r=r]rrHrFrrrrrrkr rrrr"rr6rRrrPrPrPrQsx        T,                          'JH