U &ºcuqã @s¦ddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z m Z mZmZmZddlZddlZddlmZddlmZddlmZddlmZdd lmZdd lmZmZd d l m!Z!m"Z"d d l#m$Z$d dl%m&Z&m'Z'd dl(m)Z)m*Z*m+Z+m,Z,d dl-m.Z.m/Z/zddl0m1Z1dZ2Wne3k rFdZ2YnXdd„Z4e/ƒrŽddl5m6m7Z8ddl9m:m;Z<ddl=m>m?Z@zddlAmBZBdZCWnFe3k rèzddlDmBZBdZCWne3k râdZCYnXYnXdd„ZEzFddlFZFeF G¡eFjHjIdkr"dZJeF Kd¡ne Ld¡r2dndZJWne3k rRdZJYnXdd„ZMe NeO¡ZPeQdœdd„ZReeQd œd!d"„ƒZSed#œd$d%„ZTGd&d'„d'ƒZUdS)(éN)Úcontextmanager)ÚPath)ÚCallableÚDictÚListÚOptionalÚTupleÚUnion)Únn)Ú DataLoader)ÚDataset)ÚDistributedSampler)Ú RandomSampler)ÚtqdmÚtrangeé)Ú DataCollatorÚDefaultDataCollator)ÚPreTrainedModel)ÚAdamWÚget_linear_schedule_with_warmup)ÚPREFIX_CHECKPOINT_DIRÚEvalPredictionÚPredictionOutputÚ TrainOutput)ÚTrainingArgumentsÚis_tpu_available)ÚampTFcCstS©N)Ú _has_apex©r r ú8/tmp/pip-unpacked-wheel-ymerj3tt/transformers/trainer.pyÚis_apex_available#sr")Ú SummaryWritercCstSr)Ú_has_tensorboardr r r r!Úis_tensorboard_available9sr%zZW&B installed but not logged in. Run `wandb login` or set the WANDB_API_KEY env variable.ZWANDB_DISABLEDcCstSr)Ú _has_wandbr r r r!Úis_wandb_availableJsr'©ÚseedcCs0t |¡tj |¡t |¡tj |¡dSr)Úrandomr)ÚnpÚtorchZ manual_seedZcudaZmanual_seed_allr(r r r!Úset_seedQs   r-©Ú local_rankccs.|dkrtj ¡dV|dkr*tj ¡dS)zs Decorator to make all processes in distributed training wait for the first one (locally) to do something. ©éÿÿÿÿrNr)r,Ú distributedZbarrierr.r r r!Útorch_distributed_zero_firstYs  r3©ÚdatasetcCs*t ¡dkrt|ƒSt|t ¡t ¡dS)Nr)Z num_replicasZrank)ÚxmÚxrt_world_sizerr Z get_ordinalr4r r r!Úget_tpu_sampleres r8c @sˆeZdZUdZeed<eed<eed<ee ed<ee ed<dZ ee e ge fed<eed <dZed ed <dZeejjejjjfed <dZeeed <dZeeed<dCeeeeee ee ee e ge fed eejjejjjfdœdd„Zedœdd„ZdDee edœdd„Ze edœdd„Zeeejjejjjfdœdd„Z dd „Z!e"ed!fed"œd#d$„Z#dEee$d%œd&d'„Z%dFe e$efee&dd(œd)d*„Z'e(j)e e$ej*fejjed+œd,d-„Z+edœd.d/„Z,edœd0d1„Z-dGee$d2œd3d4„Z.dHee$d2œd5d6„Z/e0dfe1e$dœd7d8„Z2dIddœd9d:„Z3dJee eee e$efd;œdd?„Z6dKee$eee5d@œdAdB„Z7dS)LÚTrainerzv Trainer is a simple but feature-complete training and eval loop for PyTorch, optimized for Transformers. ÚmodelÚargsÚ data_collatorÚ train_datasetÚ eval_datasetNÚcompute_metricsÚprediction_loss_onlyr#Ú tb_writerÚ optimizersÚ global_stepÚepochF)r:r;r<r=r>r?rArBc CsØ||_||_|dk r||_ntƒ|_||_||_||_||_| |_|dk rR||_ n"t ƒrt|jj dkrtt |jj d|_ t ƒs„t d¡tƒr”| ¡n t d¡t|jjƒ| ¡rÄtj|jjddtƒrÔd|jj_dS)a Trainer is a simple but feature-complete training and eval loop for PyTorch, optimized for Transformers. Args: prediction_loss_only: (Optional) in evaluation and prediction, only return the loss Nr0)Zlog_dirzdYou are instantiating a Trainer but Tensorboard is not installed. You should consider installing it.zœYou are instantiating a Trainer but W&B is not installed. To use wandb logging, run `pip install wandb; wandb login` see https://docs.wandb.com/huggingface.T©Úexist_ok)r:r;r<rr=r>r?r@rBrAr%r/r#Z logging_dirÚloggerÚwarningr'Ú _setup_wandbÚinfor-r)Úis_local_masterÚosÚmakedirsÚ output_dirrÚconfigZ xla_device) Úselfr:r;r<r=r>r?r@rArBr r r!Ú__init__}s8ÿ ÿ zTrainer.__init__)ÚreturncCs„|jdkrtdƒ‚tƒr$t|jƒ}n |jjdkr:t|jƒnt|jƒ}t|j|jj ||j j d}tƒr€t   ||jjg¡ |jj¡}|S)Nz+Trainer: training requires a train_dataset.r1)Ú batch_sizeÚsamplerÚ collate_fn)r=Ú ValueErrorrr8r;r/rr r Útrain_batch_sizer<Ú collate_batchÚplÚParallelLoaderÚdeviceÚper_device_loader)rPZ train_samplerÚ data_loaderr r r!Úget_train_dataloader´s"   ÿ ýüzTrainer.get_train_dataloader)r>rRcCs~|dkr|jdkrtdƒ‚|dk r&|n|j}tƒr:t|ƒnd}t|||jjd|jjd}tƒrzt   ||jj g¡  |jj ¡}|S)Nz-Trainer: evaluation requires an eval_dataset.F©rTrSÚshufflerU) r>rVrr8r r;Úeval_batch_sizer<rXrYrZr[r\)rPr>rTr]r r r!Úget_eval_dataloaderÌsûzTrainer.get_eval_dataloader)Ú test_datasetrRcCsRtƒrt|ƒnd}t|||jjd|jjd}tƒrNt ||jj g¡  |jj ¡}|S)NFr_) rr8r r;rar<rXrYrZr[r\)rPrcrTr]r r r!Úget_test_dataloaderásûzTrainer.get_test_dataloader)Únum_training_stepsrRcsˆ|jdk r|jSddg‰‡fdd„|j ¡Dƒ|jjdœ‡fdd„|j ¡Dƒddœg}t||jj|jjd }t||jj |d }||fS) a Setup the optimizer and the learning rate scheduler. We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the Trainer's init, or override this method in a subclass. NZbiaszLayerNorm.weightcs*g|]"\‰}t‡fdd„ˆDƒƒs|‘qS)c3s|]}|ˆkVqdSrr ©Ú.0Znd©Únr r!Ú sú4Trainer.get_optimizers...©Úany©rgÚp©Zno_decayrhr!Ú sz*Trainer.get_optimizers..)ÚparamsÚ weight_decaycs*g|]"\‰}t‡fdd„ˆDƒƒr|‘qS)c3s|]}|ˆkVqdSrr rfrhr r!rjsrkrlrnrprhr!rqsç)ÚlrZeps)Znum_warmup_stepsre) rBr:Znamed_parametersr;rsrÚ learning_rateZ adam_epsilonrZ warmup_steps)rPreZoptimizer_grouped_parametersÚ optimizerÚ schedulerr rpr!Úget_optimizersòs" þþû ÿzTrainer.get_optimizerscCs\t d¡tjt dd¡t|jƒdt d¡dkrXtj|j t dd¡t d|jj ƒd d S) a Setup the optional Weights & Biases (`wandb`) integration. One can override this method to customize the setup if needed. Find more information at https://docs.wandb.com/huggingface You can also override the following environment variables: Environment: WANDB_WATCH: (Optional, ["gradients", "all", "false"]) "gradients" by default, set to "false" to disable gradient logging or "all" to log gradients and parameters WANDB_PROJECT: (Optional): str - "huggingface" by default, set this to a custom string to store results in a different project WANDB_DISABLED: (Optional): boolean - defaults to false, set to "true" to disable wandb entirely z`Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"Z WANDB_PROJECTZ huggingface)ÚprojectrOZ WANDB_WATCHÚfalseZ gradientséd)ÚlogZlog_freqN) rGrJÚwandbÚinitrLÚgetenvÚvarsr;Zwatchr:ÚmaxÚ logging_steps©rPr r r!rIs  ÿzTrainer._setup_wandbzpl.PerDeviceLoader)Ú dataloaderrRcCs2tƒr$t|tjƒst‚t|jjjƒSt|jƒSdS)z\ Helper to get num of examples from a DataLoader, by accessing its Dataset. N)rÚ isinstancerYZPerDeviceLoaderÚAssertionErrorÚlenÚ_loaderr5)rPr…r r r!Ú num_examples(szTrainer.num_examples)Ú model_pathc Cs@| ¡}|jjdkr:|jj}|jjt|ƒ|jjd}n$tt|ƒ|jj|jjƒ}|jj}|j|d\}}|dk rÖtj   tj   |d¡¡rÖtj   tj   |d¡¡rÖ|  t  tj   |d¡¡¡|  t  tj   |d¡¡¡|j}| |jj¡|jjrtƒstdƒ‚tj|||jjd\}}|jjdkr6t j |¡}|jjd krdt jjj||jjg|jjd d }|jdk rš|j d |j  ¡¡|jj!|j "¡id t#ƒr´|jj$t% &¡}n,|jj$|jj|jjd krÚt j' (¡nd}t) *d¡t) *d| +|¡¡t) *d|¡t) *d|jj,¡t) *d|¡t) *d|jj¡t) *d|¡d|_-d|_.d} d} |dk rz~t| /d¡d  /d¡dƒ|_-|j-t|ƒ|jj} |j-t|ƒ|jj} t) *d¡t) *d| ¡t) *d|j-¡t) *d| ¡Wn&t0k rd|_-t) *d¡YnXd} d} | 1¡t2| t|ƒd| 3¡ d} | D]Ü}t4|d| 3¡ d}t5|ƒD]r\}}| dkrt| d8} qT| | 6|||¡7} |d|jjdksÀt|ƒ|jjkrš|dt|ƒkrš|jjræt jj7 8t 9|¡|jj:¡nt jj7 8| ;¡|jj:¡t#ƒrt% <|¡n| =¡| =¡| 1¡|j-d7_-||dt|ƒ|_.| 3¡rš|jj>dkrz|j-|jj>dks|j-dkrØ|jj?rØi}| | |jj>|d <| @¡d|d!<| } | A|¡|jjBrØ| C¡|jjDdkrš|j-|jjDdkrštE|d"ƒr|jF|jks*tG‚n||jks*tG‚tj   |jjHtI›d|j-›¡}| J|¡| K¡t  L| M¡tj   |d¡¡t  L| M¡tj   |d¡¡t) *d#|¡|jjdkrT|j-|jjkrT| N¡qÊqT|jjdkrö|j-|jjkrö|  N¡q|jjOr2t% PtQ R¡¡q2|jr$|j N¡t) *d$¡tS|j-| |j-ƒS)%a Main training entry point. Args: model_path: (Optional) Local path to model if model to train has been instantiated from a local path If present, we will try reloading the optimizer/scheduler states from there. rr)reNz optimizer.ptz scheduler.ptzQPlease install apex from https://www.github.com/nvidia/apex to use fp16 training.)Ú opt_levelr1T)Z device_idsZ output_deviceZfind_unused_parametersr;)Z metric_dictz***** Running training *****ú Num examples = %dz Num Epochs = %dz* Instantaneous batch size per device = %dzG Total train batch size (w. parallel, distributed & accumulation) = %dz" Gradient Accumulation steps = %dz Total optimization steps = %dú-ú/zE Continuing training from checkpoint, will skip to saved global_stepz# Continuing training from epoch %dz) Continuing training from global step %dz1 Will skip the first %d steps in the first epochz Starting fine-tuning.rtZEpoch)ÚdescÚdisableZ IterationÚlossrvÚmodulez+Saving optimizer and scheduler states to %szU Training completed. Do not forget to share your model on huggingface.co/models =) )Tr^r;Z max_stepsrˆÚgradient_accumulation_stepsÚintÚnum_train_epochsryrLÚpathÚisfileÚjoinZload_state_dictr,Úloadr:Útor[Úfp16r"Ú ImportErrorrZ initializeZfp16_opt_levelÚn_gpur Ú DataParallelr/ÚparallelZDistributedDataParallelrAÚadd_textZto_json_stringZ add_hparamsZto_sanitized_dictrrWr6r7r2Zget_world_sizerGrJrŠZper_gpu_train_batch_sizerCrDÚsplitrVZ zero_gradrrKrÚ enumerateÚ_training_stepÚutilsZclip_grad_norm_Z master_paramsZ max_grad_normÚ parametersZoptimizer_stepÚsteprƒZlogging_first_stepZ get_last_lrÚ_logZevaluate_during_trainingÚevaluateZ save_stepsÚhasattrr“r‡rNrÚ save_modelÚ_rotate_checkpointsÚsaveZ state_dictÚcloseÚtpu_metrics_debugÚ master_printÚmetÚmetrics_reportr)rPr‹Ztrain_dataloaderZt_totalr–rwrxr:Ztotal_train_batch_sizeZepochs_trainedZsteps_trained_in_current_epochZtr_lossZ logging_lossZtrain_iteratorrDZepoch_iteratorr§ÚinputsÚlogsrNr r r!Útrain2s  ÿÿþý  ü ÿþÿ     ÿ  ÿ  þý   "ÿÿ  " ÿ      z Trainer.train)r´ÚiteratorrRcCsˆ|jdk r|j|d<|jr>| ¡D]\}}|j |||j¡q"tƒrTtj||jdt  |d|ji–¡}|dk r||  |¡nt |ƒdS)NrD)r§r§) rDrAÚitemsZ add_scalarrCr'r~r}ÚjsonÚdumpsÚwriteÚprint)rPr´r¶ÚkÚvÚoutputr r r!r¨ås   z Trainer._log)r:r³rwrRc Cs¤| ¡| ¡D]\}}| |jj¡||<q|f|Ž}|d}|jjdkrR| ¡}|jjdkrj||jj}|jjr”t   ||¡}|  ¡W5QRXn|  ¡|  ¡S)Nrr) rµr·r›r;r[ržÚmeanr”rœrZ scale_lossZbackwardÚitem) rPr:r³rwr¼r½Úoutputsr’Z scaled_lossr r r!r¤ós    zTrainer._training_stepcCs"tƒrtjddS|jjdkSdS)NT©Úlocalr0)rr6Úis_master_ordinalr;r/r„r r r!rK s zTrainer.is_local_mastercCs0tƒrtjddS|jjdkp*tj ¡dkSdS)zƒ This will be True only in one process, even in distributed mode, even when training on multiple machines. FrÂr1rN)rr6rÄr;r/r,r2Zget_rankr„r r r!Úis_world_masters zTrainer.is_world_master)rNcCs| ¡r| |¡dS)z´ Saving best-practices: if you use default names for the model, you can reload it using from_pretrained(). Will only save from the master process. N)rÅÚ_save©rPrNr r r!r«szTrainer.save_modelcCsj|dk r |n|jj}tj|ddt d|¡t|jtƒsBt dƒ‚|j  |¡t   |jtj  |d¡¡dS)NTrEzSaving model checkpoint to %sz1Trainer.model appears to not be a PreTrainedModelztraining_args.bin)r;rNrLrMrGrJr†r:rrVZsave_pretrainedr,r­r—r™rÇr r r!rÆ$s   z Trainer._savecCs g}dd„t|jjƒ |›d¡Dƒ}|D]Z}|rJ| tj |¡|f¡q*t  d|›d|¡}|r*|  ¡r*| t |  ¡dƒ|f¡q*t |ƒ}dd„|Dƒ}|S)NcSsg|] }t|ƒ‘qSr )Ústr)rgÚxr r r!rq4sz/Trainer._sorted_checkpoints..z-*z.*z -([0-9]+)rcSsg|] }|d‘qS)rr )rgÚ checkpointr r r!rq?s) rr;rNÚglobÚappendrLr—ÚgetmtimeÚreÚmatchÚgroupsr•Úsorted)rPZcheckpoint_prefixÚ use_mtimeZordering_and_checkpoint_pathZglob_checkpointsr—Z regex_matchÚcheckpoints_sortedr r r!Ú_sorted_checkpoints1s" zTrainer._sorted_checkpointscCs†|jjdks|jjdkrdS|j|d}t|ƒ|jjkrr@rRcCs@| |¡}|j|dd}| |j¡|jjr:t t  ¡¡|jS)aÉ Run evaluation and return metrics. The calling script will be responsible for providing a method to compute metrics, as they are task-dependent. Args: eval_dataset: (Optional) Pass a dataset if you wish to override the one on the instance. Returns: A dict containing: - the eval loss - the potential metrics computed from the predictions Z Evaluation©Ú description) rbÚ_prediction_loopr¨Úmetricsr;r¯r6r°r±r²)rPr>r@Zeval_dataloaderr¾r r r!r©Qs   zTrainer.evaluatecCs| |¡}|j|ddS)zù Run prediction and return predictions and potential metrics. Depending on the dataset and your use case, your test dataset may contain labels. In that case, this method will also return metrics, like in evaluate(). Z PredictionrØ)rdrÚ)rPrcZtest_dataloaderr r r!Úpredictns zTrainer.predict)r…rÙr@rRc sŽ|dk r |n|j}|jjdkr>t|jtjjƒs>tj |j¡}n|j}| |jj ¡t ƒrd|j j j }n|j }t  d|¡t  d| |¡¡t  d|¡g}d}d}| ¡t||dD]‰t‡fdd„d Dƒƒ} ˆ ¡D]\} } |  |jj ¡ˆ| <qØt ¡B|fˆŽ} | r2| dd …\} }||  ¡ ¡g7}n| d }W5QRX|s´|dkrd| ¡ ¡ ¡}ntj|| ¡ ¡ ¡d d }ˆ d ¡dk r´|dkr®ˆd  ¡ ¡ ¡}q´tj|ˆd  ¡ ¡ ¡d d }q´t ƒrøt d|tj¡}t d|tj¡}|j dk r,|dk r,|dk r,|  t!||d¡}ni}t"|ƒd krLt |¡|d<t#| $¡ƒD]&}| %d¡sX| &|¡|d|›<qXt'|||dS)z Prediction/evaluation loop, shared by `evaluate()` and `predict()`. Works both with or without labels. Nrz***** Running %s *****rz Batch size = %d)rc3s|]}ˆ |¡dk VqdSr)Úget)rgr¼©r³r r!rj—sz+Trainer._prediction_loop..)ÚlabelsZmasked_lm_labelsér)ZaxisrßZ eval_predsZeval_out_label_ids)Ú predictionsÚ label_idsZ eval_lossZeval_)rárârÛ)(r@r;ržr†r:r,r rŸr›r[rr‰rSrGrJrŠÚevalrrmr·Zno_gradr¿rÀÚdetachÚcpuÚnumpyr+rÌrÝr6Z mesh_reduceZ concatenater?rrˆÚlistÚkeysÚ startswithÚpopr)rPr…rÙr@r:rSZ eval_lossesÚpredsrâZ has_labelsr¼r½rÁZstep_eval_lossZlogitsrÛÚkeyr rÞr!rÚxsZ        "  zTrainer._prediction_loop)NNNNFNN)N)N)N)N)N)F)NN)N)8Ú__name__Ú __module__Ú __qualname__Ú__doc__rÚ__annotations__rrrr r?rrrÚboolrArBrr,ZoptimZ OptimizerZ lr_schedulerZLambdaLRrCr•rDÚfloatrQr r^rbrdryrIr rŠrÈrµrr¨r ÚModuleZTensorr¤rKrÅr«rÆrrrÔr¬r©rrÜrÚr r r r!r9ks„   öö 7þ  4  þ    ÿ þ  ÿþr9)Vr¸ÚloggingrLr*rÎrÖÚ contextlibrÚpathlibrÚtypingrrrrrr rær+r,r Ztorch.utils.data.dataloaderr Ztorch.utils.data.datasetr Ztorch.utils.data.distributedr Ztorch.utils.data.samplerrZ tqdm.autorrZdata.data_collatorrrZmodeling_utilsrÚ optimizationrrZ trainer_utilsrrrrZ training_argsrrZapexrrrr"Ztorch_xla.core.xla_modelÚcoreZ xla_modelr6Ztorch_xla.debug.metricsÚdebugrÛr±Z%torch_xla.distributed.parallel_loaderr2Zparallel_loaderrYZtorch.utils.tensorboardr#r$Z tensorboardXr%r~Zensure_configuredÚapiZapi_keyr&Ztermwarnr€r'Ú getLoggerrírGr•r-r3r8r9r r r r!Úsr