U &ºcünã@sœdZddlZddlZddlZddlZddlZddlmZmZddl m Z ddl Z ddl mZddlmZddlmZddlmZmZmZmZdd lmZmZdd lmZd d lmZmZd d l m!Z!m"Z"eƒrÖddl#Z#e $e%¡Z&dddœZ'dddœZ(dddiiZ)dddiiZ*ddiZ+ddiZ,dZ-Gdd„de!ƒZ.Gdd„deƒZ/Gdd„de"ƒZ0Gdd„de1ƒZ2Gd d!„d!e1ƒZ3Gd"d#„d#e3ƒZ4Gd$d%„d%e1ƒZ5d&d'„Z6dS)(zn Tokenization classes for Transformer XL model. Adapted from https://github.com/kimiyoung/transformer-xl. éN)ÚCounterÚ OrderedDict)ÚOptional)Ú Tokenizer)Ú BaseTokenizer)Ú WordLevel)Ú LowercaseÚSequenceÚStripÚunicode_normalizer_from_str)ÚCharDelimiterSplitÚWhitespaceSplit)ÚBertProcessingé)Ú cached_pathÚis_torch_available)ÚPreTrainedTokenizerÚPreTrainedTokenizerFastz vocab.binz vocab.txt)Úpretrained_vocab_fileÚ vocab_filez vocab.jsonrztransfo-xl-wt103zNhttps://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-vocab.binzOhttps://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-vocab.jsonzOhttps://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-corpus.binz corpus.binc sèeZdZdZeZeZeZ gZ dddddddddddgf ‡fdd „ Z d d „Z d/d d „Z d0dd„Zdd„Zdd„Zdd„Zd1dd„Zd2dd„Zdd„Zdd„Zdd „Zd!d"„Zd#d$„Zd%d&„Zed'd(„ƒZd)d*„Zd3d+d,„Zd-d.„Z‡ZS)4ÚTransfoXLTokenizera, Transformer-XL tokenizer adapted from Vocab class in https://github.com/kimiyoung/transformer-xl This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the methods. Users should refer to the superclass for more information regarding methods. NrFúúú c  stƒjf| | | dœ| —Ž|dkr(|j}|dkr4g}tƒ|_||_||_||_||_||_ ||_ ||_ d|_ t  d |j ¡¡|_| ¡|_zL|dk rÄt |¡} |  ¡D]\}}||jkr¦||j|<q¦|dk rÔ| ¡Wn"tk røtd |¡ƒ‚YnX|dk r | ¡dS)N©Ú unk_tokenÚ eos_tokenÚadditional_special_tokensz !"#$%&()*+,-./\:;<=>?@[\]^_`{|}~z [^\s][{}]zUnable to parse file {}. Unknown format. If you tried to load a model saved through TransfoXLTokenizerFast,please note they are not compatible.)ÚsuperÚ__init__Zall_special_tokensrÚcounterÚspecialÚmin_freqÚmax_sizeÚ lower_caseÚ delimiterrÚ never_splitÚpunctuation_symbolsÚreÚcompileÚformatÚ%punction_without_space_before_patternÚ)_compile_space_around_punctuation_patternÚ%punctuation_with_space_around_patternÚtorchÚloadÚitemsÚ__dict__Ú build_vocabÚ ExceptionÚ ValueError)Úselfr!r"r#r$r%rrr&rrrÚkwargsZ vocab_dictÚkeyÚvalue©Ú __class__©úH/tmp/pip-unpacked-wheel-ymerj3tt/transformers/tokenization_transfo_xl.pyrUsLÿÿ     þÿ  zTransfoXLTokenizer.__init__cCs"d |j¡}d}t d||¡S)Nz(?=[{}])z (?=[^\s])Ú)r*r'r(r))r5Zlook_ahead_for_special_tokenZ$look_ahead_to_match_all_except_spacer;r;r<r,s zr?r@rúrzNo token in vocabulary) Úidx2symrÚsym2idxrKÚstripÚsplitÚ add_symbolÚunk_idxr4)r5rrRrTZsymbr;r;r<Ú_build_from_file­s  z#TransfoXLTokenizer._build_from_filecCsBt d¡tj |¡r*tj |td¡}n|}t |j |¡|fS)a  Save the vocabulary and special tokens file to a directory. Args: vocab_path (:obj:`str`): The directory in which to save the vocabulary. Returns: :obj:`Tuple(str)`: Paths to the files saved. z‰Please note you will not be able to load the save vocabulary in Rust-based TransfoXLTokenizerFast as they don't share the same structure.r) rEÚwarningrGrHÚisdirÚjoinÚVOCAB_FILES_NAMESr.Úsaver1)r5Z vocab_pathrr;r;r<Úsave_vocabulary¼s ÿ z"TransfoXLTokenizer.save_vocabularycCsÄ|jr:t d |j¡¡| |j¡t d t|ƒ¡¡n†t d |j|j¡¡g|_t ƒ|_ |j D]}|  |¡qd|j  |j¡D] \}}||jkr˜q¤| |¡q‚t d t|ƒt|j ƒ¡¡dS)Nzbuilding vocab from {}zfinal vocab size {}z,building vocab with min_freq={}, max_size={}z)final vocab size {} from {} unique tokens)rrErFr*r`rWr"r#rZrr[r!Ú add_specialr Ú most_commonr^)r5ÚsymZcntr;r;r<r2Ôs     zTransfoXLTokenizer.build_vocabTc Cs®|rt d |¡¡tj |¡s$t‚g}t|ddd`}t|ƒD]P\}} |rp|dkrp|ddkrpt d |¡¡|j | ||d} |  |  | ¡¡q@W5QRX|rªt   |¡}|S) Nzencoding file {} ...r>r?r@rrBrC)rDÚadd_double_eos)rErFr*rGrHrIrJrKrLrMrOÚconvert_to_tensorr.Úcat) r5rHÚorderedrPrDrjÚencodedrRrSrTrUr;r;r<Ú encode_fileès zTransfoXLTokenizer.encode_filecCsx|rt d t|ƒ¡¡g}t|ƒD]@\}}|rT|dkrT|ddkrTt d |¡¡| | |¡¡q$|rtt |¡}|S)Nzencoding {} sents ...rrBrC) rErFr*rWrLrOrkr.rl)r5rQrmrPrnrSrUr;r;r<Ú encode_sentsùs zTransfoXLTokenizer.encode_sentscCsL||jkrH|j |¡t|jƒd|j|<t|d | d¡¡|j|ƒdS)Nrz{}_idxz<>)r[rZrOrWÚsetattrr*r\©r5rir;r;r<rgs  zTransfoXLTokenizer.add_specialcCs.||jkr*|j |¡t|jƒd|j|<dS)Nr)r[rZrOrWrrr;r;r<r^ s  zTransfoXLTokenizer.add_symbolcCs2d|krt|ƒks(ntd |¡ƒ‚|j|S)z0Converts an id in a token (BPE) using the vocab.rz Index {} out of vocabulary range)rWrJr*rZ)r5rSr;r;r<Ú_convert_id_to_tokens(z'TransfoXLTokenizer._convert_id_to_tokencCsb||jkr|j|St|dƒr.|j ||j¡Sd|jkrB|jdSd|jkrV|jdStdƒ‚dS)z2 Converts a token (str) in an id using the vocab. r_rrYzHToken not in vocabulary and no token in vocabulary for replacementN)r[ÚhasattrÚgetr_r4rrr;r;r<Ú_convert_token_to_ids       z'TransfoXLTokenizer._convert_token_to_idcCsd |¡ ¡}|S)z< Converts a sequence of tokens (string) in a single string. ú )rcr\)r5ÚtokensZ out_stringr;r;r<Úconvert_tokens_to_string(sz+TransfoXLTokenizer.convert_tokens_to_stringcCst | |¡¡S©N)r.Ú LongTensorZconvert_tokens_to_ids)r5rUr;r;r<rk-sz$TransfoXLTokenizer.convert_to_tensorcCs t|jƒSrz)rWrZ©r5r;r;r<Ú vocab_size0szTransfoXLTokenizer.vocab_sizecCst|jf|jŽSrz)Údictr[Zadded_tokens_encoderr|r;r;r<Ú get_vocab4szTransfoXLTokenizer.get_vocabcCs\| ¡}|jr| ¡}|jdkr&|}n | |j¡}|rFdg|dgS|rT|dgS|SdS)Nr=zr)r\r$Úlowerr%r])r5rTrDrjrUr;r;r<Ú _tokenize7s   zTransfoXLTokenizer._tokenizecKs:d|kr |dr |j d|¡}n|j |¡r6t d¡|S)NZadd_space_before_punct_symbolrwzºYou might want to consider setting `add_space_before_punct_symbol=True` as an argument to the `tokenizer.encode()` to avoid tokenizing words with punctuation symbols to the `` token)r-Úsubr+ÚsearchrEra)r5Útextr6r;r;r<Úprepare_for_tokenizationJs ÿz+TransfoXLTokenizer.prepare_for_tokenization)FF)F)FFTF)FF)FF) Ú__name__Ú __module__Ú __qualname__Ú__doc__rdÚvocab_files_namesÚPRETRAINED_VOCAB_FILES_MAPÚpretrained_vocab_files_mapÚ&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESÚmax_model_input_sizesÚmodel_input_namesrr,rVrXr`rfr2rorprgr^rsrvryrkÚpropertyr}rrr…Ú __classcell__r;r;r9r<rHsHô8      rcs(eZdZdeedœ‡fdd„ Z‡ZS)Ú"_TransfoXLDelimiterLookupTokenizerFN)Ú normalizationc süzt||d} t| ƒ} Wn"tk r:td |¡ƒ‚YnXg} |rR| t|ƒg7} |rb| tƒg7} | tdddg7} t| ƒdkržt| ƒdkr”t | ƒn| d| _ |rªt |ƒnt ƒ| _ |rÖt||  |¡f||  |¡fƒ| _d||||||dœ} tƒ | | ¡dS) N)rz‹Unable to parse file {}. Unknown format. If you tried to load a model saved through TransfoXLTokenizer,please note they are not compatible.T)ÚleftÚrightrrZTransfoXLModel)ÚmodelrDrjrrr%Ú lowercase)rrr3r4r*r rr rWr Ú normalizerr r Z pre_tokenizerrZ token_to_idZpost_processorrr) r5rr%r—rrrDrjr“Ú tokenizerr˜Ú parametersr9r;r<rYs@   þÿ    ÿù z+_TransfoXLDelimiterLookupTokenizer.__init__)FFN)r†r‡rˆrÚstrrr‘r;r;r9r<r’Xs ÷ ÷r’cs\eZdZdZeZeZeZ gZ dddddddddddgdddf‡fdd „ Z ‡fd d „Z ‡Z S) ÚTransfoXLTokenizerFastaÐ Construct a "Fast" Transformer-XL tokenizer (backed by HuggingFace's `tokenizers` library). The Transformer-XL tokenizer is a word-level tokenizer (no sub-word tokenization). Adapted from Vocab class in https://github.com/kimiyoung/transformer-xl This tokenizer inherits from :class:`~transformers.PreTrainedTokenizerFast` which contains most of the methods. Users should refer to the superclass for more information regarding methods. NrFrrrc s8tƒjt|p|||| | | | |df| | | dœ|—ŽdS)N)rr%r—rrrDrjr“r)rrr’)r5r!r"r#r$r%rrr&rrrrDrjr“r6r9r;r<r¦s$øÿ óòzTransfoXLTokenizerFast.__init__cst d¡tƒ |¡S)Nz‚Please note you will not be able to load the vocabulary in Python-based TransfoXLTokenizer as they don't share the same structure.)rErarÚsave_pretrained)r5Zsave_directoryr9r;r<rÊsÿz&TransfoXLTokenizerFast.save_pretrained)r†r‡rˆr‰ÚVOCAB_FILES_NAMES_FASTrŠÚPRETRAINED_VOCAB_FILES_MAP_FASTrŒrrŽrrrr‘r;r;r9r<rœ•s* ñ$rœc@s<eZdZddd„Zddd„Zddd „Zdd d „Zdd„ZdS)ÚLMOrderedIteratorÚcpuNcCs€||_||_|dk r|nd|_||_| d¡||_| dd|j|¡}| |d¡ ¡  ¡  |¡|_ |j|jd|j|_ dS)zN data -- LongTensor -- the LongTensor is strictly ordered Nréÿÿÿÿr) ÚbszÚbpttÚext_lenÚdeviceÚsizeZn_stepZnarrowÚviewÚtÚ contiguousÚtoÚdataZn_batch)r5r¬r£r¤r¦r¥r;r;r<rÔszLMOrderedIterator.__init__c Cs¢|dkr|j}t||j d¡d|ƒ}||}td||jƒ}|j||…}|j|d|d|…}| dd¡ ¡ |j ¡}| dd¡ ¡ |j ¡} || |fS©Nrr) r¤Úminr¬r§Úmaxr¥Ú transposerªr«r¦) r5Úir¤Úseq_lenZend_idxZbeg_idxr¬ÚtargetÚdata_outÚ target_outr;r;r<Ú get_batchêszLMOrderedIterator.get_batchrccs0t||j d¡d|jƒD]}| |¡VqdSr­)Úranger¬r§r¤r¶)r5Ústartr±r;r;r<Úget_fixlen_iterúsz!LMOrderedIterator.get_fixlen_iterééc cs|j||}|}tj ¡dkr&|jn|jd}t|t|ttj ||¡ƒƒƒ}| ||¡\}} } || 7}|| | fV||j  d¡dkrqŒqdS)Ngffffffî?g@ré) r¤ÚnpÚrandomr®r¯ÚintÚnormalr¶r¬r§) r5r¸ZstdZmin_lenZ max_deviationÚmax_lenr±r¤r¬r³r²r;r;r<Úget_varlen_iterþs z!LMOrderedIterator.get_varlen_itercCs| ¡Srz)r¹r|r;r;r<Ú__iter__ szLMOrderedIterator.__iter__)r¡N)N)r)rrºrºr»)r†r‡rˆrr¶r¹rÂrÃr;r;r;r<r Ós     r c@s.eZdZd dd„Zdd„Zdd „Zd d „ZdS) ÚLMShuffledIteratorr¡NFcCs4||_||_||_|dk r|nd|_||_||_dS)zY data -- list[LongTensor] -- there is no order among the LongTensors Nr)r¬r£r¤r¥r¦Úshuffle)r5r¬r£r¤r¦r¥rÅr;r;r<rs zLMShuffledIterator.__init__ccsF|jrtj t|jƒ¡nt tt|jƒƒ¡}|D]}|j|Vq0dSrz)rÅr½r¾Z permutationrWr¬Úarrayr·)r5Z epoch_indicesrSr;r;r<Úget_sent_streams,z"LMShuffledIterator.get_sent_streamc csÞdg|j}t |j|j¡}t |j|j¡}d}||d… d¡| d¡d}t|jƒD]ê}d}z¾||jkr||dksŒt||ƒdkr˜t|ƒ||<tt||ƒd|j|ƒ} ||d| …|||||| …|f<||d| d…|||| …|f<||| d…||<|| 7}qdWqZt k rBd}YqFYqZXqZ|sPdS|  dd¡  ¡  |j ¡} |  dd¡  ¡  |j ¡} | | |jfVt| d¡|jƒ}|dkrÀ|| d…|d|…<| ||j| d¡¡q0dS)Nrr¢TrF)r£r.r{r¤Zfill_r·rWÚnextr®Ú StopIterationr°rªr«r¦r§r¥Zresize_) r5Ú sent_streamZstreamsr¬r³Zn_retainZ valid_batchr±Zn_filledZn_newr´rµr;r;r<Ústream_iterator$s<    ($ z"LMShuffledIterator.stream_iteratorccs"| ¡}| |¡D] }|VqdSrz)rÇrË)r5rÊÚbatchr;r;r<rÃSszLMShuffledIterator.__iter__)r¡NF)r†r‡rˆrrÇrËrÃr;r;r;r<rÄs /rÄc@s&eZdZd dd„Zdd„Zdd „ZdS) ÚLMMultiFileIteratorr¡NFcCs:||_||_||_||_|dk r$|nd|_||_||_dS)Nr)ÚpathsÚvocabr£r¤r¥r¦rÅ)r5rÎrÏr£r¤r¦r¥rÅr;r;r<r\szLMMultiFileIterator.__init__cCs.|jj|dd}|jr"tj |¡t|ƒ}|S)NT)rj)rÏrorÅr½r¾Úiter)r5rHrQrÊr;r;r<rÇhs  z#LMMultiFileIterator.get_sent_streamccsD|jrtj |j¡|jD]$}| |¡}| |¡D] }|Vq2qdSrz)rÅr½r¾rÎrÇrË)r5rHrÊrÌr;r;r<rÃps   zLMMultiFileIterator.__iter__)r¡NF)r†r‡rˆrrÇrÃr;r;r;r<rÍ[s rÍc@s2eZdZed dd„ƒZdd„Zdd„Zdd „ZdS) ÚTransfoXLCorpusNc Os>tj|f|ž|Ž}|tkr$t|}ntj |t¡}zt||d}Wn6tk rxt   d  |d t  ¡¡||¡¡YdSX||kr”t   d  |¡¡nt   d  ||¡¡|||Ž}t |¡} |  ¡D]\} } | |j| <qÂ||_|jdk rútj|jtjd|_|jdk rtj|jtjd|_|jdk r:tj|jtjd|_|S)z5 Instantiate a pre-processed corpus. )Ú cache_dirz€Corpus '{}' was not found in corpus list ({}). We assumed '{}' was a path or url but couldn't find files {} at this path or url.z, Nzloading corpus file {}z'loading corpus file {} from cache at {})Zdtype)rÚfrom_pretrainedÚPRETRAINED_CORPUS_ARCHIVE_MAPrGrHrcÚ CORPUS_NAMErÚEnvironmentErrorrEÚerrorr*ÚkeysrFr.r/r0r1rÏÚtrainZtensorÚlongÚvalidÚtest) ÚclsZpretrained_model_name_or_pathrÒÚinputsr6rÏZ corpus_fileZresolved_corpus_fileÚcorpusZ corpus_dictr7r8r;r;r<rÓ|s>  úÿ       zTransfoXLCorpus.from_pretrainedcOs(t||Ž|_d|_d|_d|_d|_dSrz)rrÏÚdatasetrÙrÛrÜ)r5Úargsr6r;r;r<r¨s  zTransfoXLCorpus.__init__cCsÂ||_|jdkrT|j tj |d¡¡|j tj |d¡¡|j tj |d¡¡nH|jdkrv|j tj |d¡¡n&|jdkrœtj |ddd ¡}t |¡}|j ¡|jd kr|jjtj |d¡d d |_ |jjtj |d¡d d |_ |jjtj |d¡d d |_ n¶|jd krp|jjtj |d¡d dd|_ |jjtj |d¡d dd|_ |jjtj |d¡d dd|_ nN|jdkr¾||_ |jjtj |d¡dd d|_ |jjtj |d¡dd d|_ dS)N)ÚptbÚwt2Úenwik8Útext8z train.txtz valid.txtztest.txtÚwt103Úlm1bz41-billion-word-language-modeling-benchmark-r13outputz'training-monolingual.tokenized.shuffledz news.en-*)rârãræT)rm©räråF)rmrD)rmrj) ràrÏrVrGrHrcÚglobr2rorÙrÛrÜ)r5rHràZtrain_path_patternZ train_pathsr;r;r<Ú build_corpus¯s8   ü      zTransfoXLCorpus.build_corpuscOs¦|dkrP|jdkr&t|jf|ž|Ž}q¢|jdkr¢d|d<t|j|jf|ž|Ž}nR|dkr¢|dkrf|jn|j}|jdkrˆt|f|ž|Ž}n|jdkr¢t|f|ž|Ž}|S)NrÙ)rârãrærärårçTrÅ)rÛrÜrÛ)ràr rÙrÍrÏrÛrÜrÄ)r5r]rár6Z data_iterr¬r;r;r<Ú get_iteratorÑs    zTransfoXLCorpus.get_iterator)N)r†r‡rˆÚ classmethodrÓrrêrër;r;r;r<rÑ{s  +"rÑc Cstj |d¡}tj |d¡}tj |¡r>t d¡t |¡}nÒtj |¡rvt d¡t|dƒ}t  |¡}W5QRXnšt d  |¡¡i}|dkr¦dg|d <d |d <nP|d krÂdg|d <d |d <n4|dkrîg|d <d |d <tj |d¡|d<n|dkröt ||f|Ž}t  ||¡|S)Nzcache.ptz cache.pklzLoading cached dataset...z%Loading cached dataset from pickle...ÚrbzProducing dataset {}...)rærãrr!Fr$râTrçz1b_word_vocab.txtrrè) rGrHrcrIrErFr.r/rKÚpickler*rÑre)ÚdatadirràÚfnZ fn_picklerßÚfpr6r;r;r<Ú get_lm_corpusâs2           rò)7r‰réÚloggingrGrîr(Ú collectionsrrÚtypingrZnumpyr½Z tokenizersrZtokenizers.implementationsrZtokenizers.modelsrZtokenizers.normalizersrr r r Ztokenizers.pre_tokenizersr r Ztokenizers.processorsrZ file_utilsrrZtokenization_utilsrrr.Ú getLoggerr†rErdržr‹rŸrrÔrÕrr’rœÚobjectr rÄrÍrÑròr;r;r;r<Ús`        ÿÿÿÿÿÿ=>;M g