U <c=9@sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlZddlmZddlmZmZddlmZmZmZd\ZZdZdZd Zd \ZZd Zed ed iZ d Z!edediZ"edediedediededid Z#ej$e%Z&dddd Z'ej(ej)ej*dZ+ddZ,dZ-ddZ.ddZ/ddZ0ddZ1d d!Z2d"d#Z3d$d%Z4e5d&d'd(Z6d4d*d+Z7d,d-Z8d.d/Z9d0d1Z:e;d2kre,Zk rYnXdS)5aEnd-to-end example to test a PR for regressions: $ python -m examples.end_to_end --pr 39850 $ python -m examples.end_to_end --pr 39967 $ python -m examples.end_to_end --pr 39744 NOTE: This example assumes that you have and environment prefixed with `ref_`, and another prefixed with `pr_` for the PR in question. (e.g. `ref_39850` and `pr_39850`). A helper script (examples/prepare_e2e.sh) is provided to build the required environments with the correct configuration. N)unary)Timer Measurement)DictTupleList)main subprocesszpr_{pr}zref_{pr})398503996739744)cpugpu @FT)int8float32float64)rr)rrrcCst}|jdttdtd|jdtdd|jddd|jd tttftd |jd ttt fdd |jd tdd|jd tdd|jdtdd| }|j dkrt j |_ |S)Nz--prr)typedefaultchoicesz --num_gpus)rrz--test_variance store_true)actionz--DETAIL_context)rrrz--DETAIL_devicez --DETAIL_envz--DETAIL_result_filez --DETAIL_seed)argparseArgumentParser add_argumentstr_PR_LISTint_MAIN _SUBPROCESS_CPU_GPU parse_argsnum_gpustorchcudaZ device_count)parserargsr,M/tmp/pip-unpacked-wheel-gikjz4vx/torch/utils/benchmark/examples/end_to_end.pyr&Rs  r&zsource activate {source_env} && python -m examples.end_to_end --pr {pr} --DETAIL_context subprocess --DETAIL_device {device} --DETAIL_env {env} --DETAIL_result_file {result_file} --DETAIL_seed {seed}c s|dkrfdddD\}}}}tjd}|jd|d}|||g|}tttd|jdt|dd } d |d | d d fS|dkrdS|dkrtjd}|jddd} d| d dfSt ddS)Nr csg|] }|qSr,r,.0iparamsr,r- rsz,construct_stmt_and_label..)k0k1k2dimZ random_valuer)lowhighrztorch.topk(x, dim=z, k=)Ztopkr )z torch.std(x)Zstdr r7ztorch.sort(x, dim=sortz Unknown PR) nprandomZ RandomStaterandintmaxr!flooruniformlog2 ValueError) prr2r4r5r6r7stateZtopk_dimZdim_sizekZsort_dimr,r1r-construct_stmt_and_labelps(rHcCs|j}|jtk}t|jd}t|jD]}t|}tj |||d t }t |D]~\}\}} } || d<t |j| \} } t| || d|d|d|d| |jd} | jtd }| | d |_t|t||qRq(W5QRXdS) Nab)seeddtyper) dtype_str[z, seed=z] (z ), stmt = )stmtglobalslabel descriptionenv)Z min_run_time)tensor_parametersr2)Z DETAIL_seedZ DETAIL_devicer%openZDETAIL_result_file_DTYPES_TO_TESTrE_DTYPE_STR_TO_DTYPErZ UnaryOpFuzzerZtake_RUNS_PER_LOOP enumeraterHrZ DETAIL_envZblocked_autorange _MIN_RUN_SECmetadataprintpickledump)r+rJr)frLrKiteratorr0ZtensorsrSr2rNrPZtimer measurementr,r,r-subprocess_mains8  rac siii}}jtjdtjdftjtrdt<tjD]}t |qNt j j|t<fddtt tDtt}|tt||t<tjtr dt<tt d}t j ||t<fddtt tDtt}|tt||t<g}|D]F}|D]:}||ddD} tdd | d d q4q,t|D]} | qt|jdS) NrErcsg|]}|djfqS)T test_variancer/rJr+envsfinished_countsrEr,r-r3sz_main..rcsg|]}|djfqS)Frcrerfr,r-r3sc Ss0g|](\}}|d|dt|t|qS)z: z / ) _NUM_LOOPS _REPLICATESr/rGvr,r,r-r3s z )end)rE_REF_ENV_TEMPLATEformat_PR_ENV_TEMPLATE_DEVICES_TO_TESTr%ranger'_AVAILABLE_GPUSputmultiprocessingdummyZPoolrirjimapmap_fnr$r! cpu_countvaluesappenditemsr[joincloseprocess_resultsrd) r+poolsZ map_itersr0ZtrialsZ cpu_workersresultsZmap_iterrprogresspoolr,rfr-_mainsH        rcs|sdSdd|DdD]*dkr0q"tfddDs"tq"dd|D}t|d krp|nd d}fd d|D}ttj||d <|d <dd |d <tf|S)NcSsg|] }|qSr,) __getstate__r/mr,r,r-r3szmerge..r)number_per_runtimesrZc3s"|]}|dkVqdS)rNr,)r/s)rGstatesr,r- szmerge..cSsh|] }|jqSr,rrr,r,r- szmerge..rcs"g|]fddjDqS)csg|]}|jqSr,r)r/t)rnr,r-r3sz$merge...)r)r/)r)rr-r3srrrZ) keysallAssertionErrorlenpoplistitchainr)Z measurementsZnumbers_per_runZ merged_staterr,)rGrrr-mergesrc s6i}|D]`\\}}}|D]N}|j|j|j||f}||ggg|jdrPdnd}||||qqdd|D}t|D],\}\} } t dd| | fDr |qfdd|D}t t d t |d d d |D} d d |D} | r t | d|| r2t | d|dS)NrefrrcSs&i|]\}\}}|t|t|gqSr,)r)r/keyZ r_ref_listZ r_pr_listr,r,r- s z#process_results..css|]}|dkp|jVqdSN)Z has_warnings)r/rr,r,r-rsz"process_results..csi|]\}}|kr||qSr,r,rkZflagged_for_removalr,r-rsz samples were culled, z remaincSs g|]\}}|dr||fqSrr,rkr,r,r-r3sz#process_results..cSs g|]\}}|ds||fqSrr,rkr,r,r-r3sZCPUZGPU)rPrQZ num_threads setdefaultrR startswithr}r~setanyaddr[rconstruct_table) rrdZpaired_resultsrJuse_gpuZ result_batchrrindexr_refr_prZ gpu_resultsZ cpu_resultsr,rr-rs0   rc Cs`d|d|rdndddd}tdd |d dd td d |Dd d d}t|}tdd|D}tdd|D}|||}dddg}t||||gD]4\}} t|dd| dd| t|dddqdd|ddDdd|t|d d!t|d d!DBd"d|d#d$DB} |d%d&|t|d d'd&h} d(dd)d*dd+d,dd-d.dd-d/dd0d1d2 } tjd3d4\} }t|d5}| |d6| d td | d7|D]X\}\}}}t ||j |j |}| |d || krt||| krtd8qtd9W5QRXtt d:td;|dS).cSs|dS)Nr:r,)r0r,r,r-z!construct_table..)rcSsg|]}|ddkr|qS)r:g?r,r.r,r,r-r3s z#construct_table..cSsg|]}|ddkr|qS)r:gr,r.r,r,r-r3s zImproved (>5%):zRegressed (>5%):z Within 5%:z<17>6z (dz>3.0fz%)cSsh|] }|dqSrr,r.r,r,r-rsz"construct_table..cSsh|] }|dqSrr,r.r,r,r-rsr:rcSsh|] }|dqSrr,r.r,r,r-rsiiru-Relative Δ Absolute Δ | numel>8rKz>14shape>10stepsZlayoutz>7ztask specific z~==============================================================================================================================z.logsuffixwtz z1 [First twenty omitted (these tend to be noisy) ]z...z/[Last twenty omitted (these tend to be noisy) ]a steps: Indicates that `x` is sliced from a larger Tensor. For instance, if shape is [12, 4] and steps are [2, 1], then a larger Tensor of size [24, 4] was created, and then x = base_tensor[::2, ::1]. Omitted if all elements are ones. layout: Indicates that `x` is not contiguous due to permutation. Invoking `x.permute(layout)` (e.g. x.permute((2, 0, 1)) if layout = [2, 0, 1]) would produce a Tensor with physical memory layout matching logical memory layout. (Though still not contiguous if `steps` contains non-one elements.) z Complete results in: )ljustr[sortedrzipr!tempfilemkstemprTwriterow_strrtextwrapdedent)rZ device_strrdrZ n_regressedZ n_improvedZ n_unchangedZlegendsZlegendcountZ keys_to_printZellipsis_afterZ column_labels_Zresult_log_filer^rrrrel_diffrowr,r,r-r sJ"  .,$0    rc sr|jd|jd}d}|dd}fddt|D}|dd}ttd d t|t|Drld n|}d }|jd r|jdd dd d\} } | d| d}n0|jdrn"|jdr|jdd dd }|dddt |dddd dd|dddddtfd dt|Dd!d"td#d |DsVt|nd dd"|dd d$|S)%Nr2rSr7xZnumelcsg|]}d|qS)Zx_step_r,r.r1r,r-r3Jszrow_str..ordercss|]\}}||kVqdSrr,)r/r0jr,r,r-rLszrow_str..rnz torch.topkztorch.topk(x, z, z<8z torch.stdz torch.sortztorch.sort(x, rz>5.0fz% g.Az>11.1fz usr|z>12z rLrcsg|]}d|qS)rGr,r.r1r,r-r3Xsz>17rcss|]}|dkVqdSrr,r.r,r,r-rXsr) rZrtrrrrNrreplacesplitabs) rZ diff_secondsr`rSr7Zx_numelrrZ task_specificZdim_strZk_strr,r1r-rDs"    (    r) result_filec CsNg}t|d6}z|t|Wqtk r<Yq@YqXqW5QRX|S)Nrb)rTr}r\loadEOFError)routputr^r,r,r- read_resultscs rrncCs&tj|t|tdddtjddS)NPATHrn)ZCUDA_VISIBLE_DEVICESrT)rRstdoutshell)r runrosgetenvPIPEcmdZcuda_visible_devicesr,r,r-rns rcCs6|D],}td|}|jdkrtd|dqdS)zEnsure that subprocesszsource activate rzFailed to source environment ``N)r returncoderD)rgrRresultr,r,r- test_sourcezs rc Cs|\}}}}}}|rtnd}zztjdd\} }|D]B} t j |rL|dn| | ||rZt nt ||d} t | |rr|nddq8||rt nt d7<||ft|fWWStk rYnXW5|dk rt|tj|rt|XdS)Nz.pklrr)Z source_envrRrEZdevicerrJrnrr)rugetrvrpathexistsremoverr_SUBPROCESS_CMD_TEMPLATErqr%r$rrKeyboardInterrupt) r+rJrgrErrhrdrrrrRrr,r,r-rzs,    rzcCs,ttj|jdtj|jdgt|dS)Nrb)rrprqrErrr)r+r,r,r-rs   r__main__r )rn)?__doc__r itertoolsrrwZmultiprocessing.dummyrr\queuer rrZnumpyr=r(Z torch.utils.benchmark.op_fuzzersrZtorch.utils.benchmarkrrtypingrrrr"r#rrrpr r$r%rYrjrWrirsQueuer!rurUrrrrVr&rrHrarrrrrrrrrrzr__name__r+ZDETAIL_contextrr,r,r,r-s       .#;