U <c@s~ddlZddlZddlZddlmZddlZddlZddlmZddl m Z ddl m Z ddlm mZddlm mZddlZddlmZddlmZddlmZddlmZmZddlmZmZmZmZdd l m!Z!dd l"m#Z#d d d d ga$d d d d ga%e&a'ej(d d ddZ)ddZ*ddZ+dIddZ,ddZ-ddZ.d dej/fddZ0ej1j2ej3dddZ4dd Z5d!d"Z6d#d$Z7ej1j2d%d&Z8ej1j2eej3ej3ej3d'd(d)Z9d*d+Z:d,d-Z;d.d/Zd5d6Z?Gd7d8d8eZ@Gd9d:d:eZAGd;d<dd>eBZCGd?d@d@eBZDGdAdBdBeBZEGdCdDdDe!ZFGdEdFdFejGZHGdGdHdHe!ZIdS)KN)Enum) timedelta)Function)once_differentiable)RRef)IS_MACOSsandcastle_skip_if) dist_init initialize_pgwait_until_node_failure worker_name)RpcAgentTestFixture)skip_if_lt_x_gpuFT requires_gradcCsdt|<|t|<t|dSNT)rpc_donectx_idsknown_context_idsadd)ctx_id rank_distancer^/tmp/pip-unpacked-wheel-gikjz4vx/torch/testing/_internal/distributed/rpc/dist_autograd_test.py _set_rpc_done.srcCst|stdqdS)Ng?)rtimesleep)rrrr_check_rpc_done7srcCstj||dS)Nrtorchones)Zsizesrrrr _torch_ones<sr#cCsNt|}||}|jr8|js&t|}|}n |jrBtt||SN) dist_autograd get_gradients local_valueZ is_sparseAssertionErrorZto_denser!equal) context_idrrefgradgradsxrrr_compare_owner_valueAs     r/cCstjdddSNrrTrr rrrr create_tensorMsr2cCsBdddgdddgg}dddg}tj||d||d}|r>|}|S) Nrg @gffffff@g333333@r1rdtype)r!Zsparse_coo_tensorcoalesce)r7rr6ivtensorrrrbuild_sparse_tensorQs  r;)returncCstdS)Nr1)r!r"Zrequires_grad_rrrrcreate_torchscript_tensorZsr=cCs t||Sr$r!rt1t2rrr my_py_add_srBcCs||Sr$r)abrrr my_scalar_addcsrEcCst||}|Sr$)r!rr')rref_t1rAretrrr my_rref_addgsrHcCs t||Sr$r>r?rrr my_script_addlsrI)ref_t1rAr<cCs|}t||Sr$)to_herer!r)rJrAr@rrrmy_script_ref_addqsrLcCstj|t||fdSNargs)rpcrpc_syncrH)dstrFrArrrmy_nested_rref_addwsrScCstSr$)requires_grad_tensorrrrrret_requires_grad{srUc CsR|d|}|dkr6tjt|t|||||dfdStjt|t||fdSdS)Nr3rrN)rPrQr my_py_nested_callrB)r@rArR world_sizeZhopsZnext_dstrrrrVs rV c Cslt}t}t||kr`|tkr`tD]2}zt|Wq*tk rZ||Yq*Xq*q|tk}|Sr$)rsetrr%_retrieve_context RuntimeErrorr)Ztimeout_secondsstartZcontext_id_to_raisedr*successrrr_all_contexts_cleaned_upsr^c Csxtf}tj|t||fd}|r0tj|}n|}t||gtj|t ||fdtj|t ddW5QRXdSNrNr) r%contextrPrQrHr!sparsesumbackwardrrrFrAZps rank_diffrbr*rGlossrrr _run_trainers rhc Csxtf}tj|t||fd}|r0tj|}n|}t||gtj|t ||fdtj|t ddW5QRXdSr_) r%rarPrQrLr!rbrcrdrrrerrr_run_trainer_torchscripts ric@s,eZdZdZeddZeeddZdS)SimulateBackwardErrorTcCs|Sr$rctxinputrrrforwardszSimulateBackwardError.forwardcCstjrtdn|SdS)NSimulate error on backward pass)rj_simulate_error Exceptionrkrrrrds zSimulateBackwardError.backwardN)__name__ __module__ __qualname__rp staticmethodrnrrdrrrrrjs  rjc@seZdZdZdZdZdZdS)ExecModer3r4rN)rrrsrtLOCALRPC_SYNCREMOTE RPC_ASYNCrrrrrvsrvc@seZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ d9ddZddZddZddZd d!Zd"d#Zd$d%Zd&d'Zd(d)Zd*d+Zd,d-Zd.d/Zd0d1Zd2d3Zd4d5Zd6d7Zd8S):CommonDistAutogradTestcGstj|kr8t|dkr0t|dtr0||dS||Stj|krVtjt|||dStj |krxtj t|||d Stj |krtj t|||d}|Std|dS)Nr3rrNUnrecognized ExecMode {})rvrxlen isinstancelistryrPrQr rzremoterKr{ rpc_asyncwait ValueErrorformat)selfrR exec_modemethodrOZfutrrr_exec_func_with_dsts     z*CommonDistAutogradTest._exec_func_with_dstcGs|j|||f|Sr$)r _next_rank)rrrrOrrr _exec_funcsz!CommonDistAutogradTest._exec_funccCsJt|dr2|jd|j|_|j|jkrD|Sn|jd|j|_|jS)Ndst_rankr3)hasattrrrWrankrrrrrrs    z!CommonDistAutogradTest._next_rankcCs t|dSr$)r)rrrrrrsz&CommonDistAutogradTest._check_rpc_donecGs<|tjkr$tj|dd|DS|j|||f|dS)NcSsg|] }|jqSrr,).0argrrr sz.)rvrxr!autogradrd_verify_backwards_remote)rrtensorsr* local_gradsrOrrr_verify_backwardss  z(CommonDistAutogradTest._verify_backwardsc Gst||t|}t|}d}td|D]R}||dk rn|||||||||||d7}q,||||q,||t|dS)Nrr3)r%rdr&r~rangeassertIn assertEqual assertNotIn) rrr*rrOr-nargsZngradsr8rrrrs    z/CommonDistAutogradTest._verify_backwards_remotec Cs|jd|j}t|j|j|jtj}|r@t}t}n tjdddd}tj dddd}t j |krt j t||||fd}n6t j|krt jt||||fd}ntd|t j t|t|dfdt} ||| | } |dt| | } |dt| |t| dt| d||||dtt d} | } |dt| |!t| dt"#W5QRX|$t%t|} W5QRX|$t%t} W5QRXdS)Nr3rTrrNr}r)&rrWr file_init_methodr%rar;r!r"zerosrvryrPrQr rzrrKrrr_current_contextr _context_id_send_functionsr~_recv_functions _verify_graph_for_first_rpc_callrvaluesrrZr_verify_graph_for_rpc_call_execdistbarrier assertRaisesr[) rfnrrbrr*r@rArGrlsend_functionsrecv_functionsrrr _test_graphsZ       z"CommonDistAutogradTest._test_graphc Cs|jd|j}t|j|j|jt}|rHtdd}tdd}n tjdddd}tj dddd}|d|j}t j |krt j t|t||||jdfd}n>t j|krt jt|t||||jdfd}ntd|tdD](} t j t|j| |jt|| fdqtt} ||| | } |dt| | } |dt| |t| dt| d|||t!t"d} |#| t!t"d } |#| t!t"d} | } |dt| |$t| dtW5QRXdS) Nr3TrrrNr})r3r4rrr4)%rrWr rr%rar;r!r"rrvryrPrQr rVrzrrKrrrrrrrrrr~rrrrrZr!_verify_graph_for_nested_rpc_callr) rrrbrr*r@rAZ nest_dst_rankrGrdrlrrrrr_test_graph_for_py_nested_callJsh          z5CommonDistAutogradTest._test_graph_for_py_nested_callc Cs|jd|j}t|j|j|jt}|rHtdd}tdd}n tjdddd}tj dddd}t j |krt j t|t|||jd|j|j|jdfd}nPt j|krt jt|t|||jd|j|j|jdfd}ntd|t j t|jd|jt|dfdt}||||} |dt| |} |dt| |t| dt| d||||t| d| dt!t"d}|#|t$%W5QRXdS) Nr3TrrrrNr}r4)&rrWr rr%rar;r!r"rrvryrPrQr rVrzrrKrrrrrrrr~rrrrrrrZrrrr rrrbrr*r@rArGrlrrrrr%_test_graph_for_py_nested_call_itselfsl         z}|r8t|ddkd}ntjdd|ddkd}||q|}tj |krt j t |tj |fd} n6tj|krt jt |tj |fd} ntd||t || ttdj} d} tt| D]6}|d| |d|||| |djqt} | } |t| d || |hW5QRXdS) NrXr4rrrrNr}torch::autograd::AccumulateGradr3)r%rarr;r!r"appendrrvryrPrQr stackrzrrKrrrrrrrnext_functionsr~namevariable_known_worker_ids)rrrbr*Z num_tensorsrr8r:rrG next_funcsidxrl worker_idsrrr_test_rpc_complex_argssN      z-CommonDistAutogradTest._test_rpc_complex_argsFc stjjj|r<jdj}|dj}|h}nfddtjD}tX}|D]L}tjt |||dtjt |t |dfd|rbtjt |t |dfdqbW5QRX t t |W5QRXtt}|dS)Nr3csh|]}|jkr|qSrrrrrrr Gs zECommonDistAutogradTest.context_cleanup_test_helper..rNr4r rrrWrr%rarPrQr rrr[rZrrr^ assertTrue) rrpc_argsfuncnestedrZnested_dst_rank dst_ranksr*r]rrrcontext_cleanup_test_helper:s2  z2CommonDistAutogradTest.context_cleanup_test_helperc Cst}tjt|tj||fd}|r:tj |}n| }tj ||gdd| |j | |j t||}|rtj |}n| }| | |j | |j |j }|j }t ||g|||j |||j W5QRXdSNrNTZ retain_graph)r%rarPrQr rr!rrbrcrdZ assertIsNoner,assertIsNotNoner) rr@rArbr*rgZ loss_localZt1_grad_beforeZt2_grad_beforerrr_backward_no_grad_on_tensor_s0       z2CommonDistAutogradTest._backward_no_grad_on_tensorc Cst||}|rtj|}n|}|t}|rNtj|t dd} ntj|t dddid} ||krtj|t | |fd} ntj|t || |fd} | } |rtj| } n| } t|| gt|} ||| || ||j|tj|t|| |jfdW5QRXdS)N)FTrN)r1rTrOkwargs)r!rrbrcrdr%rarPrr;r#rHrSrKr&rrr,rrQr/) rcallee rref_ownerr@rArrb local_retr*rFr+rGr-rrr_backward_rrefsP     z%CommonDistAutogradTest._backward_rrefc Cs<|rtdd}tdd}ntjddd}tjddd}t||}|rXtj|n |tj t |j |dd}dddg}g} |D]:} | tj t |j | |j|||t |j | |fdq|D]} || qd} |D]<} t| } t| } |}||| || ||jqtdd tj| dS) NTrr1rrNr3r4rr)r;r!r"rrrbrcrdrPrr rrrrWrrr%r&rKrrr,rfuturesZwait_all)rZ create_ref_fnZ trainer_fnrbr@rArrFZ rank_diffsrrfZaccumulate_grad_funcrr-Zlocal_t1rrr_test_trainer_pssF          z'CommonDistAutogradTest._test_trainer_psc CstjtjtjfD]}t} ||tj||} ||tj || } ||tj || f} ||tj || f} |r||tj | | } ||tj | | } tj | } n,||tj | | } ||tj| | } | } ||| g| |||||| }|r|n|}W5QRXqdSr$)rvrxryrzr%rarr!rmulrrbrcZbmmmatmulr)rr@rAt3t4t5rrbrr*vals1s2rgrGrrr_backward_multiple_round_tripss2 z5CommonDistAutogradTest._backward_multiple_round_tripsc Cspd}tjtjfD]Z}tH}||tj||}|rBtj |}n| }| ||g||||}W5QRXqdSr$) rvrxrzr%rarr!rrbrcr)rr@rArbrrr*rgrrr_backward_different_dtypess z1CommonDistAutogradTest._backward_different_dtypesc Csnd}tjtjfD]X}tF}||t||}|r@tj |}n| }| ||g||||}W5QRXqdSr$) rvrxrzr%rarrBr!rbrcr) rr@rArbrrr*rGrgrrr_backward_simple_python_udfs z2CommonDistAutogradTest._backward_simple_python_udfc Csd}tjtjtjtjfD]d}tR}||t||}|rHt j |}n| }| ||g||||} | rn| n|}W5QRXqdSr$) rvrxryr{rzr%rarrIr!rbrcr) rr@rArbrrr*Z forward_retrgrGrrr_backward_simple_script_call s( z3CommonDistAutogradTest._backward_simple_script_callc Csvtd}tjt|tj|||fd}|r@tj |}n| }tj ||gddt ||gW5QRXdSr) r%rarPrQr rDistAutogradTest&_test_nested_backward_accumulate_gradsr!rbrcrd)rr@rArbr*rGrgrrr!_nested_backward_accumulate_grads3s   z8CommonDistAutogradTest._nested_backward_accumulate_gradsc Cs||}||}||}|||||}|r>tj|}n|}tj|gt}tjt | t j ||| fd}|rtj|}n|}t||gt |} ||j| |||j| |W5QRXdSrM)r!rbrcrrdr%rarPrQr rr_nested_python_udfr&rr,) rr@rArbrrresrgr*r-rrr_backwards_nested_python_udfBs*    z3CommonDistAutogradTest._backwards_nested_python_udfc CstjtjfD]}t}||tj||}|||||rNt j |}n| }t ||g| |j||jt|}||||||||||W5QRXq dSr$)rvryrzr%rarr_mixed_requires_grad_operatonrr!rbrcrdrr assertFalser&rr) rr@rArbrr*rGrgr-rrr_mixed_requires_grad]s&      z+CommonDistAutogradTest._mixed_requires_gradc Cspt^}tjt|tj||fd}|r:tj |}n| }t dD]}tj ||gddqJW5QRXdS)NrNTr) r%rarPrQr rr!rrbrcrrd)rr@rArbr*rgr8rrr_multiple_backwardps   z)CommonDistAutogradTest._multiple_backwardcCs|j}|dt||d|dd|||ddj|d|dd|d|dd|||ddj|d|dd||j|dS)Nr4rrr3)rrr~rrgrad_fn)r send_functionZ recv_functionr@rArGrrrrrsz7CommonDistAutogradTest._verify_graph_for_first_rpc_callc Cs~tjtjtjfD]h}tV}|||tj||}|rDtj |} n| } | || g||||}|rj|n|}W5QRXqdSr$) rvrxryrzr%rarr!rrbrcr) rrRr@rArrbrr*rGrgrrr_backward_simples* z'CommonDistAutogradTest._backward_simplecCs|j}|dt||dd}|d||j}|dt||d|dd|d|dd||dd|dddS)Nr3rZ AddBackward0r4-torch::distributed::autograd::RecvRpcBackward)rrr~r)rrrZadd_backward_fnrrrrs z6CommonDistAutogradTest._verify_graph_for_rpc_call_execcCs|}|dt|t|dj}|dt||d|dd|d|dd||dd|ddt|dj}|dt||d|dddS)Nr4rrr3)rrr~rrrr)rrlrrrrrrs&z8CommonDistAutogradTest._verify_graph_for_nested_rpc_callN)F)rrrsrtrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr|s6 <SJ*) %$45r|c@seZdZeddZeddZeddZeddZed d Zed d Z ed dZ eddZ eddZ eddZ eddZeddZeddZeddZeddZedd Zed!d"Zed#d$Zed%d&Zed'd(Zed)d*Zed+d,Zed-d.Zed/d0Zed1d2Zed3d4Zed5d6Zed7d8Zed9d:Z ed;d<Z!ed=d>Z"d?S)@TensorPipeAgentDistAutogradTestcCs|tjtjddSrrr!rrvryrrrr"test_graph_for_builtin_call_sparseszBTensorPipeAgentDistAutogradTest.test_graph_for_builtin_call_sparsecCs|ttjddSrrrBrvryrrrr!test_graph_for_python_call_sparseszATensorPipeAgentDistAutogradTest.test_graph_for_python_call_sparsecCs|tjtjddSrrr!rrvrzrrrr)test_graph_for_builtin_remote_call_sparseszITensorPipeAgentDistAutogradTest.test_graph_for_builtin_remote_call_sparsecCs|ttjddSrrrBrvrzrrrr(test_graph_for_python_remote_call_sparseszHTensorPipeAgentDistAutogradTest.test_graph_for_python_remote_call_sparsecCs|tjddSrrrvryrrrr$test_graph_for_py_nested_call_sparseszDTensorPipeAgentDistAutogradTest.test_graph_for_py_nested_call_sparsecCs|tjddSrrrvrzrrrr+test_graph_for_py_nested_remote_call_sparse szKTensorPipeAgentDistAutogradTest.test_graph_for_py_nested_remote_call_sparsecCs|tjddSrrrvryrrrr+test_graph_for_py_nested_call_itself_sparse szKTensorPipeAgentDistAutogradTest.test_graph_for_py_nested_call_itself_sparsecCs|tjddSrrrvrzrrrr2test_graph_for_py_nested_remote_call_itself_sparseszRTensorPipeAgentDistAutogradTest.test_graph_for_py_nested_remote_call_itself_sparsecCs|tjddSrrrvryrrrr2test_no_graph_with_tensors_not_require_grad_sparseszRTensorPipeAgentDistAutogradTest.test_no_graph_with_tensors_not_require_grad_sparsecCs|tjddSrrrvrzrrrr9test_no_graph_with_tensors_not_require_grad_remote_sparseszYTensorPipeAgentDistAutogradTest.test_no_graph_with_tensors_not_require_grad_remote_sparsecCs|tjddSrrrvryrrrrtest_rpc_complex_args_sparseszZ"ed?d@Z#edAdBZ$edCdDZ%edEdFZ&edGdHZ'dIdJZ(edKdLZ)edMdNZ*edOdPe+e,dQdRdSZ-edTdUZ.edVdWZ/edXdYZ0edZd[Z1ed\d]Z2ed^d_Z3ed`daZ4e5dbdcZ6edddeZ7e5dfdgZ8e5dhdiZ9edjdkZ:dOZ;edOdPe+e,dQdldmZdrZ?Gdsdtdte@ZAedudvZBeCdwdxZDeCdydzZEeCd{d|ZFed}d~ZGGddde@ZHeddZIe5ddZJeddZKeddZLe5ddZMeddZNeddZOedOdPddZPeddZQeddZReddZSeddZTeddZUe5ddZVeddZWdrS)rc Csd}|||jd>tg}tdD]J}t8}||t|||j|d?||W5QRXq*|D],}| t d |t|W5QRXqzdS)Nl0+Could not find autograd context with id: {}) rZ worker_idr%Z _get_max_idrrarZrrassertRaisesRegexr[r)rZmax_auto_incrementZ context_idsr8r*rrrtest_autograd_contexts(    z&DistAutogradTest.test_autograd_contextc CsDt2}|tdt}W5QRXW5QRXW5QRXdS)Nz3Already have an autograd context id for this thread)r%rar7r[)rr*rrrtest_nested_contexts  z$DistAutogradTest.test_nested_contextcCs|tjtjddSNFrrrrrtest_graph_for_builtin_callsz,DistAutogradTest.test_graph_for_builtin_callcCs|ttjddSr:rrrrrtest_graph_for_python_callsz+DistAutogradTest.test_graph_for_python_callcCs|tjtjddSr:rrrrr"test_graph_for_builtin_remote_call#sz3DistAutogradTest.test_graph_for_builtin_remote_callcCs|ttjddSr:rrrrr!test_graph_for_python_remote_call'sz2DistAutogradTest.test_graph_for_python_remote_callcCs|tjddSr:rrrrrtest_graph_for_py_nested_call+sz.DistAutogradTest.test_graph_for_py_nested_callcCs|tjddSr:rrrrr$test_graph_for_py_nested_remote_call/sz5DistAutogradTest.test_graph_for_py_nested_remote_callcCs|tjddSr:rrrrr$test_graph_for_py_nested_call_itself3sz5DistAutogradTest.test_graph_for_py_nested_call_itselfcCs|tjddSr:rrrrr+test_graph_for_py_nested_remote_call_itself7sz.rFrrNr3T)rrWr%rar!r"rrPrQr rrrrrr) rrr*r@rArrlrrGrrrtest_worker_ids_recordeds:  z)DistAutogradTest.test_worker_ids_recordedc stn}tjdddd}tjdddd}tjt|tj||fd }tj j }t ||gW5QRXW5QRX|jfdd}|d}|d}|d } ||jd ||jd || j|j|| j|jdS) NrTrrNcsfddDdS)Ncsg|]}|jkr|qSr)r)reventZ partial_keyrrrs zTDistAutogradTest.test_dist_autograd_profiling..get_event..rrrRfunction_eventsrRr get_eventsz@DistAutogradTest.test_dist_autograd_profiling..get_eventZSendRpcBackwardZRecvRpcBackwardz&torch::distributed::autograd::backwardr3)r%rar!r-rPrQr rrrcrZprofilerZprofilerdrTrcountZ assertGreaterZcpu_time_total) rr*r@rArgprUZ send_eventZ recv_eventZbackward_eventrrSrtest_dist_autograd_profilings "" z-DistAutogradTest.test_dist_autograd_profilingc CsltZ}tjdddd}tjdddd}|t$tjt| tj ||fdW5QRXW5QRXdS)NrTrrN) r%rar!r-rr[rPrQr rr)rr*r@rArrrtest_error_in_contexts   z&DistAutogradTest.test_error_in_contextcCs&|tjdddtjdddddSNr1TrF)rr!r-rrrrtest_backward_no_grad_on_tensors   z0DistAutogradTest.test_backward_no_grad_on_tensorcCs.||tjdddtjddddddSr[)rrr!r-rrrrtest_backward_simples  z%DistAutogradTest.test_backward_simplecCs,||jtjdddtjddddddSr[)rrr!r-rrrrtest_backward_simple_selfs  z*DistAutogradTest.test_backward_simple_selfc Cs<t|}|}|||tjdddtjddddddSr[)r rrr!r-rrrrtest_backward_rrefs   z#DistAutogradTest.test_backward_rrefc Cs>|jdkr:d}|}|||tjdddtjddddddS)Nrrr1TrF)rrr!r-rrrrtest_backward_rref_multis   z)DistAutogradTest.test_backward_rref_multic CsTt|jd|j}t|jd|j}|||tjdddtjddddddS)Nr3r4r1TrF)r rrWrr!r-rrrrtest_backward_rref_nesteds  z*DistAutogradTest.test_backward_rref_nestedcCs|ttddSr:)rr2rhrrrrtest_trainer_pss z DistAutogradTest.test_trainer_pscCs0ddlmmm}d|_|ttddS)NrTF)Ztorch.distributed.rpc.api distributedrPapiZ_ignore_rref_leakrr=ri)rrdrrr%test_trainer_ps_torchscript_functionssz6DistAutogradTest.test_trainer_ps_torchscript_functionsc CsD|tjdddtdtjdddtdtjddddddSr[)rr!r-rrrr"test_backward_multiple_round_trips%s   z3DistAutogradTest.test_backward_multiple_round_tripsc Csd}tjddd}td}tjddd}td}tjtjtjfD]r}t`}||tj ||}||tj j |||f}| } | || g||||||| } | r| n|}W5QRXqDdS)N)rwrYTr)rYr))r))rgr*)r!r-rvrxryrzr%rarrlinalg multi_dotrcr) rrr@rArrrr*rrgrGrrr#test_backward_different_tensor_dims1s,   z4DistAutogradTest.test_backward_different_tensor_dimsc Csd}tjddd}tjddd}tjddd}tjtjtjfD]}tr}||tj |||f}||tj t |dddt |ddd}| } | || g|||||} | r| n|}W5QRXq>dS)Nr1Trrr3r4)r!r-rvrxryrzr%rarrrZnarrowrcr) rrr@rArrr*srrgrGrrrtest_backward_unused_tensorsDs0 z-DistAutogradTest.test_backward_unused_tensorsc Csd}tjddd}tjtjtjfD]}tp}||tj |d}|d}|d}|d}||tj j |||f} | } | || g|||} | r| n|}W5QRXq"dS)N)rXr4Trr4rrw)r!r-rvrxryrzr%rarsplitrhrircr) rrtrr*Z tensor_listr@rArrrgrGrrr%test_backward_multiple_output_tensorsZs$ z6DistAutogradTest.test_backward_multiple_output_tensorsc Cspt^}tjddd}tjddd}tjt|tj||fd}t ||}t || gW5QRXdS)Nr1TrrN) r%rar!r-rPrQr rrrrdrc)rr*r@rArrrrr1_run_test_backward_unused_send_function_in_threadms   zBDistAutogradTest._run_test_backward_unused_send_function_in_threadcCs8tj|jd}d|_||d||dS)NtargetTrX) threadingThreadrpdaemonr\joinris_alive)rrnrrr"test_backward_unused_send_function}s z3DistAutogradTest.test_backward_unused_send_functionc Cst}tjddd}tjddd}||||}t|}tjt| tj ||fd}tjt| tj ||fd}tjt| tj ||fd}tjt| tj ||fd}|tdt||gW5QRXW5QRXdS)Nr1TrrNz5Error on Node [0-9]+: Simulate error on backward pass)r%rar!r-rjapplyrPrQr rrrrdivr7r[rdrc)rr*r@rAtmprrrrr#test_backward_autograd_engine_errors<      z4DistAutogradTest.test_backward_autograd_engine_errorF)Zclean_shutdownzITest is flaky on MacOS since libuv error handling is not as robust as TCPc Cstdt|j|j|jt}tj ddd}tj ddd}tj t | tj ||fd}t|jddkr|}t|jD]}|ddkrt||q|t|t||gW5QRXnW5QRXdS)Nr)r1TrrNr4r)rP_set_rpc_timeoutr rrrWr%rar!r-rQr rrrrget_shutdown_error_regexrr r7r[rdrc)rr*r@rArshutdown_error_regexrrrrtest_backward_node_failures&     z+DistAutogradTest.test_backward_node_failurec Csrtjddd}tjddd}d}|td|6tjt|tj ||fd}t || gW5QRXdS)Nr1Trdr6rN) r!r-r7r[rrPrQr rrr%rdrc)rr@rAr*rrrrtest_backward_without_contexts z.DistAutogradTest.test_backward_without_contextc Cs|j}t}tjddd}tjddd}t||}t||gt|}| dt || ||| ||| t dd||| t dd||W5QRXdS)Nr1Trr4r) rr%rar!r-rrdrcr&rr~rr")rrr*r@rArr-rrrtest_backward_without_rpcs     z*DistAutogradTest.test_backward_without_rpcc Cs"t}|tdt|dW5QRX|tdtddW5QRX|tdt|gW5QRX|td tdd}t||gW5QRX|td$tjdddd}t||gW5QRX|td"tjd dd}t||gW5QRXW5QRXdS) Nzincompatible function argumentsz,No tensors provided for gradient computationzrequires_grad not set onrz,is not a scalar, all roots need to be scalarTrz'does not have a valid gradient functionr3)r%rar7 TypeErrorrdr[r!r-)rr*rnrrrtest_backward_invalid_argss2  z+DistAutogradTest.test_backward_invalid_argsc Csd}tjddd}tjddd}tjtjfD]}tx}||tj|| }||tj || }||tj | }||tj || } | ||||| g||||}W5QRXq,dSr0)r!r-rvrxryr%rarrrcrcosrzr) rrr@rArr*Zr1Zr2Zr3Zr4rrrtest_backward_multiple_roots s"  z-DistAutogradTest.test_backward_multiple_rootscCs.|tjddtjdtjddtjdddS)Nr1Tr5F)rr!r-rrrrrrtest_backward_different_dtypess z/DistAutogradTest.test_backward_different_dtypescCs*|tjddddtjddddddSNrTrF)rr!r-rrrrtest_backward_simple_python_udf#s z0DistAutogradTest.test_backward_simple_python_udfcCs*|tjddddtjddddddSr)rr!r-rrrr test_backward_simple_script_call+s z1DistAutogradTest.test_backward_simple_script_callcCsFtjj||}tjj||}tjj||}tj|||||gSr$)r!r, functionalZlinearrhri)r@rArrrrrr_complex_python_udf3sz$DistAutogradTest._complex_python_udfc Cszd}tjddd}tjddd}tjtjfD]H}t6}||tj ||}| }| ||g||||}W5QRXq,dSr0) r!r-rvrxrzr%rarrrrcr)rrr@rArr*rGrgrrr test_backward_complex_python_udf:s( z1DistAutogradTest.test_backward_complex_python_udfcCs&||}t|}tj||||gSr$)rjryr!rhri)r@rArrrrr_python_udf_with_backward_errorKs z0DistAutogradTest._python_udf_with_backward_errorcCs<||}||}tjt|tj||fd}tj|||gSrM)rPrQr rrr!rhri)r@rArRrrrr_nested_rpc_call_backward_errorQsz0DistAutogradTest._nested_rpc_call_backward_errorc Cstjddd}tjddd}tT}tjt|tj |||fd}| t dt || gW5QRXW5QRXdS)Nr1TrrNro)r!r-r%rarPrQr rrrr7r[rdrc)rr@rAr*rgrrrtest_backward_python_udf_error\s   z/DistAutogradTest.test_backward_python_udf_errorc Cs tdt|j|j|jt}tj ddd}tj ddd}| }tj t |t ||||jdfd}t|jdkrW5QRdStj}|jdkr|}td||t|t||gW5QRX|d d n|d gtd d W5QRXdS) Nr)r1Trr3rNr4rZ0test_backward_node_failure_python_udf_rank0_doneTruerX)seconds)rPr}r rrrWr%rar!r-rrQr rVrrZdistributed_c10dZ_get_default_storer~r r7r[rdrcrYrr)rr*r@rArRrstorerrrr%test_backward_node_failure_python_udfms,      z6DistAutogradTest.test_backward_node_failure_python_udfcCs<||}||}tjt|t||fd}|||||SrM)rPrQr rB)r@rArRrrrrrrrsz#DistAutogradTest._nested_python_udfcCs*|tjddddtjddddddSr)rr!r-rrrr test_backwards_nested_python_udfs z1DistAutogradTest.test_backwards_nested_python_udfNc@s(eZdZeddZeeddZdS)zDistAutogradTest.MyBackwardFunccCs|Sr$rrkrrrrnsz'DistAutogradTest.MyBackwardFunc.forwardcCs0tjdk sttttjts,t|Sr$)r'_test_clean_context_backward_context_idr(rrr%Z_release_contextr^rkrrrrds z(DistAutogradTest.MyBackwardFunc.backwardNrrrsrtrurnrrdrrrrMyBackwardFuncs  rc CsFt|j|j|jt}|}|t_t d|jD]<}||jkr4||j|j|j}t j t |t ||fdq4t||jdtttjddd}t ddD]&}|}t j t |tj||fd}qtj|}|dt|d}|td|t||gW5QRXtt jd d t !dd S) a This test simulates the situation where the 'backward' call might throw an exception locally which would lead to the autograd context being cleaned up if we're using the context manager. As a result, the autograd context might be cleaned up while some threads are still using the autograd context. It is fine for the 'backward' call to throw an exception in this test, but the process should not crash. rrNr3r1Trrr6F)ZgracefulN)"r rrrWr%Z _new_contextrrrrrPrQr rrrrr~rr!r-rrrryrr7r[rrdrcshutdownsysexit)rrar*r8rr@rRrrr"test_clean_context_during_backwards:     z3DistAutogradTest.test_clean_context_during_backwardcCs|}||||Sr$)r')clsembedding_rrefrmr2r1 embeddingrrrr/sz'DistAutogradTest._call_remote_embeddingcCs|}t|}||jSr$)r'r%r&r.)rrr*rZgrad_maprrrr0s zDistAutogradTest._get_gradcCs|jr||S||SdSr$r)rr@rArrrr sz.DistAutogradTest._mixed_requires_grad_operatoncCs*|tjddddtjddddddSr)rr!r-rrrrtest_mixed_requires_grads z)DistAutogradTest.test_mixed_requires_gradc@s(eZdZeddZeeddZdS)z"DistAutogradTest.TestDebugInfoFunccCs|Sr$rrkrrrrnsz*DistAutogradTest.TestDebugInfoFunc.forwardcCs8t}|dk stt|d}|dkr0|dks4t|S)Nnum_current_backward_passesr3rw)r%_get_debug_infor(int)rlrm debug_infoZbackward_passesrrrrds   z+DistAutogradTest.TestDebugInfoFunc.backwardNrrrrrTestDebugInfoFuncs  rc Cst|j|j|jtjddd}tjddd}t}d}i}|||<t|jD]<}||jkrTt j t |tj |||fd||d<|d7}qTt j||||d<|d7}t|jD]<}||jkrt j t |tj |||fd||d<|d7}qt|||gt}t|d}||dko8|dkW5QRXt|jdD]2} t j t |j| d|jt|| dfdqTtt}|dk st|dt|d |t|d k|tt}|dt|ddS) Nr1TrrrNr3Znum_autograd_contextsrwrr4)r rrrWr!r-r%rarrPrQr rrrryrdrcrrrrrrr(rr~r^) rr@rAr*r8rrrZnum_autograd_contextrrrrtest_debug_info+sV        "  z DistAutogradTest.test_debug_infoc Cstjddd}tjddd}tp}tjdtj||fd}tjdtj||fd}tjdtj||fd}tjdtj||fd}t || gW5QRXdS)Nr1TrrrN) r!r-r%rarPrQrrrrdrc)r@rAr*rrrt6rrr_workload_threades z!DistAutogradTest._workload_threadcCslt|j|j|j|jdkr`g}tdD]$}tjtjd}| | |q(|D] }| qRt dS)z This test ensures async processing for distributed autograd works appropriately. This is achieved by spawning multiple threads and hammering a single node with a lot of backward() calls. rrqN)r rrrWrrsrtrrr\rrvrr)rthreadsr8rnthreadrrrtest_async_dist_autogradqs    z)DistAutogradTest.test_async_dist_autogradc Cstjddd}tjddd}t}t||}tjj|gddtj|gtj t | tj||fd}tj||gddt||gt |}| dt|||||||| |j||| |j||W5QRXdS)Nr1TrrrNr4)r!r-r%rarrrdrcrPrQr rr&rr~rr,rr@rAr*rr-rrrtest_backward_accumulate_gradss&      z/DistAutogradTest.test_backward_accumulate_gradscCstjt|tj||fdSrM)rPrQr r!r)r@rArrrrrsz7DistAutogradTest._test_nested_backward_accumulate_gradscCs*|tjddddtjddddddSr)rr!r-rrrr%test_nested_backward_accumulate_gradss z6DistAutogradTest.test_nested_backward_accumulate_gradscCs*|tjddddtjddddddSr)rr!r-rrrrtest_multiple_backwards z'DistAutogradTest.test_multiple_backwardc Cst|j|j|jtjddd}tjddd}t}tj d | t j ||fd}zxtdD]j}|dkr|tdtj||gdd W5QRXqf|dkrtj||gdd qftd t_tqfW5tdt_XW5QRXdS) Nr1Trzworker{}rNr2rorF)r rrrWr!r-r%rarPrQrrrrrcrrrjrprr7r[rd)rr@rAr*rgr8rrr"test_multiple_backward_with_errorss*    z3DistAutogradTest.test_multiple_backward_with_errorsc Cstjddd}|ddtjddd}d}tjtjtjfD]T}tB}| |tj ||}| }| ||g||||}|r|n|}W5QRXq>dS)Nr1TrcSs|dSNr4rrrrrz=DistAutogradTest.test_backward_verify_hooks..) r!r" register_hookrvrxryrzr%rarrrcr)rr@rArrr*rGrgrrrtest_backward_verify_hookss" z+DistAutogradTest.test_backward_verify_hooksc sGfdddtGfdddtGdddt}tjddd d }tjddd d }t\}t||||gt|}||| j k||| j kW5QRXtL}t||d d gt|}j }|| }| ||kW5QRXt}t|||d d gt|}j }|| }|| }|||k||| j k||| j kW5QRXd S)z6 Similar to test in test_autograd.py. cs,eZdZdZeddZefddZdS)z2DistAutogradTest.test_no_grad_copy..MyFuncNcSs||Sr$rrlinp1Zinp2rrrrnsz:DistAutogradTest.test_no_grad_copy..MyFunc.forwardcs|_||fSr$data_ptrstatic_grad_ptrrlr,MyFuncrrrds z;DistAutogradTest.test_no_grad_copy..MyFunc.backwardrrrsrtrrurnrdrrrrrs  rcs,eZdZdZeddZefddZdS)z.MyFuncSingleGradNcSs|Sr$rrlinprrrrnszDDistAutogradTest.test_no_grad_copy..MyFuncSingleGrad.forwardcs|_|Sr$rrMyFuncSingleGradrrrds zEDistAutogradTest.test_no_grad_copy..MyFuncSingleGrad.backwardrrrrrrs  rc@s$eZdZeddZeddZdS)z;DistAutogradTest.test_no_grad_copy..NonContGradFunccSs||_tdgS)Ng?)sizer!r:)rlrrrrrn s zCDistAutogradTest.test_no_grad_copy..NonContGradFunc.forwardcSstd|jSNr3)r!r"expandrrrrrrd szDDistAutogradTest.test_no_grad_copy..NonContGradFunc.backwardN)rrrsrtrurnrdrrrrNonContGradFunc s rr)rYTrr3rN) rr!randnr%rardryr&rrrr) rrrCrDr*r-p_gp_ap_br)rrrtest_no_grad_copys2     "       z"DistAutogradTest.test_no_grad_copyc sGfdddtGfdddttjdddd}tjdddd}td d d d d dd d g}tdd g}ddlmm}t} |}|j |||dd }tj ||gddt |} j} | |} || | ktdD]} tj ||gddqW5QRXt} ||}|j |||dd }tj ||gddt |} j} | |} | |} || | k|| | k|| | ktdD]} tj ||gddqW5QRXdS)Ncs,eZdZdZeddZefddZdS)z9DistAutogradTest.test_no_grad_copy_sparse..MyFuncNcSs|Sr$rrrrrrn2 szADistAutogradTest.test_no_grad_copy_sparse..MyFunc.forwardcs|_|Sr$)_valuesrrrrrrrd6 szBDistAutogradTest.test_no_grad_copy_sparse..MyFunc.backwardrrrrrr/ s  rcs,eZdZdZeddZefddZdS)zBDistAutogradTest.test_no_grad_copy_sparse..NonContGradFuncNcSs||Sr$rrrrrrn> szJDistAutogradTest.test_no_grad_copy_sparse..NonContGradFunc.forwardcsftdd}tjddtjd}|dd}|dd}tj||tddg}| _ ||fS)Nr3r)r6r+rX) r!r-r"longrrbZ FloatTensorSizerrr)rlr,r9r8nvniZngradrrrrdB s   zKDistAutogradTest.test_no_grad_copy_sparse..NonContGradFunc.backwardrrrrrr; s  rrXrTrr3r4rwr)r*rrbr)rr!rr:torch.nn.functionalr,rr%rary embedding_bagrcrdr&rrrrrr)rrCrDrmr2Fr* emb_matrixrgr-rrr8rr)rrrtest_no_grad_copy_sparse, s<         z)DistAutogradTest.test_no_grad_copy_sparsec sGfdddttjdddd}tddd d d ddd g}td d g}d dlmm}t~} |}|j |||dd  }tj ||gddt |}j} ||} |j|j|| | kW5QRXdS)Ncs4eZdZdZdZdZeddZefddZdS)zHDistAutogradTest.test_grad_copy_sparse_indices_extra_ref..MyFuncNcSs|Sr$rrrrrrn~ szPDistAutogradTest.test_grad_copy_sparse_indices_extra_ref..MyFunc.forwardcs&|_|_|_|Sr$)rrrZ_indicesstatic_grad_indices_refstatic_grad_values_refrrrrrd s  zQDistAutogradTest.test_grad_copy_sparse_indices_extra_ref..MyFunc.backward) rrrsrtrrrrurnrdrrrrry s rrXrTrr3r4rwr)r*rrr)rr!rr:rr,rr%raryrrcrdr&rrrrrrr) rrCrmr2rr*rrgr-rrrrr'test_grad_copy_sparse_indices_extra_refv s     z8DistAutogradTest.test_grad_copy_sparse_indices_extra_refc sd_fdd}fdd}tjdddd}||}|jjdd}|||||jjd d}||tR}|}t ||g d jt |} d t |  || kW5QRXdS) Nrcsjd7_|Srhook_called_timesZ output_gradsZ input_gradsrrrpost_hook_add_one sz;DistAutogradTest.test_post_hooks..post_hook_add_onecsjd7_|Srrrrrrpost_hook_add_two sz;DistAutogradTest.test_post_hooks..post_hook_add_tworXTrr3r))rr!r-rrrr%rarcrdrr&r~r) rrrrnrCZaccumulate_grad_0Zaccumulate_grad_1r*rgr-rrrtest_post_hooks s"       z DistAutogradTest.test_post_hookscCstd||}d|_|S)Nr3T)rrr)r@rArrrr _slow_add s zDistAutogradTest._slow_addc Cstd}td}||}d|_|t|jd|j}tj |t j ||fd}t >}|}t ||g|tj|t|||jfdW5QRXdS)Nr1Tr3rN)r!r-rrcrdr rrWrPrrrr%rarKrrQr/r,)rr@rArrRr+r*rgrrrtest_thread_local_context_id s"      z-DistAutogradTest.test_thread_local_context_id)Xrrrsrtr r8r9r;r<r=r>r?r@rArBrCrDrErFrGrHrIrLrMrNrOrPrXrZr\r]r^r_r`rarbrerfrjrlrorprxr|rrrrrrrrrrrurrrrrZ_backward_donerrrrrrr classmethodr/r0rrrrrrrrrrrrrrrrrrrrrrrs@                                                   (   <     9         H I (  rc@sHeZdZededdZededdZededdZdS) CudaDistAutogradTestr3c Cstjddddd}tjddddd}||t^}||}t||gt|}|dt|||j ||||j ||W5QRXdS)NrTcuda:0rdevicer4) r!r-rcrdr%rar&rr~r,rrrrtest_gpu_simple s  z$CudaDistAutogradTest.test_gpu_simplec Cstjddddd}tjdddd}tdD]}d|_d|_d}tjtjfD]}t}| |tj ||}| d|}| |tj | |} | d|} | |tj | | } | || g||||} | r| n|}W5QRXqJq*dSNrTrrrrr!r-rr,rvrxryr%rarrZcudacpurrc) rr@rAr8rrr*rrrrZt7rGrrrtest_gpu_to_cpu_continuation s,  z1CudaDistAutogradTest.test_gpu_to_cpu_continuationc Cstjddddd}tjdddd}tdD]}d|_d|_d}tjtjfD]}tp}| |tj ||}| d|}| |tj | |} | d|} | || g||||} | r| n|}W5QRXqJq*dSrr) rr@rAr8rrr*rrrrrGrrr%test_gpu_to_cpu_continuation_gpu_root s*  z:CudaDistAutogradTest.test_gpu_to_cpu_continuation_gpu_rootN)rrrsrtrr rrrrrrrr s  rc@s,eZdZddZeddZeddZdS)FaultyAgentDistAutogradTestc stjjjfddtjD}t<}|D]0}tjt |||dtjt |t |dfdq8W5QRX t t |W5QRXtt}|dS)Ncsh|]}|jkr|qSrrrrrrr, s zJFaultyAgentDistAutogradTest.context_cleanup_test_helper..rNr3r)rrrrr*rr]rrrr$ s  z7FaultyAgentDistAutogradTest.context_cleanup_test_helpercCs8tjdddd}tjdddd}|j||ftjddSrJrKr rrrrL@ szAFaultyAgentDistAutogradTest.test_context_cleanup_tensor_with_gradcCsL||jtjjj||jjd||jjd|t |jj ddS)Nr+rrw) r rpc_backendrPZbackend_registryZ BackendTypeZFAULTY_TENSORPIPErpc_backend_optionsZnum_worker_threadsZnum_fail_sendsr~Zmessages_to_failrrrrtest_verify_backend_optionsF sz7FaultyAgentDistAutogradTest.test_verify_backend_optionsN)rrrsrtrr rLrrrrrr! s  rcs,eZdZfddZddZddZZS) WrapperModulecst|||_dSr$)super__init__tomodel)rrr __class__rrrO s zWrapperModule.__init__cGs |j|Sr$)r)rrOrrrrnS szWrapperModule.forwardcs"t|fdd|jDS)Ncsg|] }|qSrrrrWr-rrrX sz+WrapperModule.gradients..)r%r&r parameters)rrrrr gradientsV s zWrapperModule.gradients)rrrsrtrrnr __classcell__rrrrrN s rc@sdeZdZedddZGdddejjZGdddejjZ eddd Z edd d Z d S) TensorPipeCudaDistAutogradTestrwc Cs"|j}t|jd|j}|||j|jd|jitjt|j|j|j|j|dtj d|jdd}tj d|jdd}t }tj |tj ||fd}t ||gt |}|td|||td||||j||j||j||jW5QRXtdS)Nr3rbackendrrWrrXTrrrN)rr rrWset_device_maprPinit_rpcrr!r-r%rarQrrdrcr&rr"rr)roptionsrRr@rAr*rr-rrrtest_device_maps_backward_pass] s*  z=TensorPipeCudaDistAutogradTest.test_device_maps_backward_passcs$eZdZfddZddZZS)z.TensorPipeCudaDistAutogradTest.MyRemoteComputecstdSr$)rrrrrrr{ sz7TensorPipeCudaDistAutogradTest.MyRemoteCompute.__init__cCs |d}|S)N@rrrmrrrrn~ sz6TensorPipeCudaDistAutogradTest.MyRemoteCompute.forwardrrrsrtrrnrrrrrMyRemoteComputez s r cs$eZdZfddZddZZS)z-TensorPipeCudaDistAutogradTest.MyLocalComputecst||_dSr$)rr next_stage)rr rrrr s z6TensorPipeCudaDistAutogradTest.MyLocalCompute.__init__cCs|j|Sr$)r rQrnr rrrrn sz5TensorPipeCudaDistAutogradTest.MyLocalCompute.forwardr rrrrMyLocalCompute s rc Cs|j}t|jd|j}|||j|jd|jitjt|j|j|j|j|dt|t j }t |}t dD]}t jddg|jdd}|d}t}||} | tD} ||}||} t| | gt| } ||j| |W5QRXqxtdS) Nr3rrXri'Trr )rr rrWrrPrrrrr rrr!r-randomrcrdr%rar&rr,r) rrrRZremote_computeZ local_compute_rmresultrrgr*r-rrrtest_dist_autograd_sync_streams s2      z>TensorPipeCudaDistAutogradTest.test_dist_autograd_sync_streamsc Cs|j}t|jD]}|t||j|iqtjt|j|j|j|j|d|jdkrddt|jdD}dd|D}g}td|jD]*}| tj t|t ||d|fdqt dd d}tj|}||t} |D]} | |}qt| |gg} |D]} | | | q"tt| D]F} d d|| D} t| | | D]\}}|||qvqJW5QRXtdS) NrrcSsg|]}tddqS))r,ZLinear)rrrrrr szRTensorPipeCudaDistAutogradTest.test_gradients_synchronizations..r3cSsg|]}|dqSr`)r)rlrrrr srNircSsg|] }|jqSrrrrrrr s)rrrWrr rrPrrrrrr!rrr,Z Sequentialrcrdr%rarQrnrrr~rziprrr)rrZ peer_rankZlayersZ local_layersZ remote_layersrr.Z local_modelr*Z remote_layerZfutsr8Zlocal_gradientsZg1Zg2rrrtest_gradients_synchronizations sF    z>TensorPipeCudaDistAutogradTest.test_gradients_synchronizationsN) rrrsrtrrr!r,Moduler rrrrrrrr[ s  &r)F)rX)Jrrsrenumrrr!Ztorch.nnr,datetimerZtorch.distributedrcrZtorch.distributed.autogradrr%Ztorch.distributed.rpcrPZ"torch.testing._internal.dist_utilsZtorch.autogradrZtorch.autograd.functionrrZ$torch.testing._internal.common_utilsrrr r r r Z>torch.testing._internal.distributed.rpc.rpc_agent_test_fixturer Z*torch.testing._internal.common_distributedrrrrYrr"rTrrr#r/r2rr;ZjitscriptZTensorr=rBrErHrIrLrSrUrVr^rhrirjrvr|rrrrrrrrrrrs                 #u?-