U &ch!8@sdZddlZddlZddlZddlZddlmZee Z dddZ dd idd idZ dd iZ d d ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;dd?d@dAdBdC7ZdDdEZGdFdGdGeZdS)Hz)Tokenization classes for Salesforce CTRL.N)PreTrainedTokenizerz vocab.jsonz merges.txt) vocab_file merges_fileZctrlzHhttps://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.jsonzHhttps://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txtiiiiii#jivi~i6iivii.iiiךiͨii%iii3iR-iniS.iKiiwii[i*iii/i?iin1iipiiiiϒi i)i-i(iiKiiiǢiihi)7Z PregnancyZ ChristianityZExplainZFitnessZSavingZAskZAssZJokeZ QuestionsZThoughtsZRetailZFeminismZWritingZAtheismZNetflixZ ComputingZOpinionZAloneFunnyZGamingZHumanZIndiaZJokerZDietZLegalZNormanZTipZWeightZMoviesZRunningZScienceZHorrorZ ConfessionZFinanceZPoliticsZScaryZSupportZ TechnologiesZTeenageEventZLearnedZNotionZ WikipediaZBooksZExtractZ ConfessionsZ ConspiracyZLinksZ NarcissusZ RelationshipZ RelationshipsZReviewsZNewsZ TranslationZ multilingualcCs>t}|d}|ddD]}|||f|}qt|}|S)zReturn set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings). rrN)setadd)wordpairsZ prev_charcharrB/tmp/pip-unpacked-wheel-ymerj3tt/transformers/tokenization_ctrl.py get_pairsfsrcsveZdZdZeZeZeZ e Z dfdd Z e ddZddZd d Zd d Zd dZddZddZddZZS) CTRLTokenizera Constructs a CTRL tokenizer. Peculiarities: - Byte-Pair-Encoding This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the methods. Users should refer to the superclass for more information regarding methods. Args: vocab_file (:obj:`str`): Path to the vocabulary file. merges_file (:obj:`str`): Path to the merges file. unk_token (:obj:`string`, `optional`, defaults to ""): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. c stjfd|i|t|dd}t||_W5QRXdd|jD|_t|dd}| ddd}W5QRXd d |D}t t |t t ||_i|_dS) N unk_tokenutf-8encodingcSsi|]\}}||qSrr).0kvrrr sz*CTRLTokenizer.__init__.. rcSsg|]}t|qSr)tuplesplit)rmergerrr sz*CTRLTokenizer.__init__..)super__init__openjsonloadencoderitemsdecoderreadrdictziprangelen bpe_rankscache)selfrrrkwargsZ vocab_handleZ merges_handleZmerges __class__rrr"s zCTRLTokenizer.__init__cCs t|jSN)r-r&r0rrr vocab_sizeszCTRLTokenizer.vocab_sizecCst|jf|jSr4)r*r&Zadded_tokens_encoderr5rrr get_vocabszCTRLTokenizer.get_vocabc s|jkrj|St|}tt|dd|ddg}t|}|sN|St|fddd}|jkrpqn|\}}g}d}|t|krDz|||} Wn,tk r| ||dYqDYnX| ||| | }|||kr,|t|dkr,||d|kr,| |||d7}q| |||d7}qt|}|}t|dkrdqnqNt|}qNd |}|dd }|j|<|S) Nrzcsj|tdS)Ninf)r.getfloat)pairr5rrz#CTRLTokenizer.bpe..keyrr@@ ) r/rlistrminr.r-index ValueErrorextendappendjoin) r0tokenr r ZbigramfirstsecondZnew_wordijrr5rbpesF  "  2      zCTRLTokenizer.bpecCs>g}td|}|D]$}|dd||dDq|S)z Tokenize a string. z\S+\n?cSsg|]}|qSrr)rtrrrr sz+CTRLTokenizer._tokenize.. )refindallrGrOr)r0textZ split_tokenswordsrJrrr _tokenizes  "zCTRLTokenizer._tokenizecCs|j||j|jS)z2 Converts a token (str) in an id using the vocab. )r&r9r)r0rJrrr_convert_token_to_idsz"CTRLTokenizer._convert_token_to_idcCs|j||jS)z=Converts an index (integer) in a token (str) using the vocab.)r(r9r)r0rErrr_convert_id_to_tokensz"CTRLTokenizer._convert_id_to_tokencCsd|dd}|S)z< Converts a sequence of tokens (string) in a single string. rQrA)rIreplacestrip)r0tokensZ out_stringrrrconvert_tokens_to_stringsz&CTRLTokenizer.convert_tokens_to_stringc Cstj|s td|dStj|td}tj|td}t|ddd}| t j |j dd W5QRXd }t|dddh}| d t |jd d dD]@\}}||krtd||}| d|d|d7}qW5QRX||fS)a Save the vocabulary and special tokens file to a directory. Args: save_directory (:obj:`str`): The directory in which to save the vocabulary. Returns: :obj:`Tuple(str)`: Paths to the files saved. z*Vocabulary path ({}) should be a directoryNrrwrrF) ensure_asciirz#version: 0.2 cSs|dS)Nrr)kvrrrr<r=z/CTRLTokenizer.save_vocabulary..r>zqSaving vocabulary to {}: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!rQrr)ospathisdirloggererrorformatrIVOCAB_FILES_NAMESr#writer$dumpsr&sortedr.r'warning) r0Zsave_directoryrZ merge_filefrEwriterZ bpe_tokensZ token_indexrrrsave_vocabularys*   zCTRLTokenizer.save_vocabulary)r)__name__ __module__ __qualname____doc__rgZvocab_files_namesPRETRAINED_VOCAB_FILES_MAPZpretrained_vocab_files_map&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESZmax_model_input_sizes CONTROL_CODESZ control_codesr"propertyr6r7rOrVrWrXr]rn __classcell__rrr2rrus  , r)rrr$loggingraregexrRZtokenization_utilsr getLoggerrordrgrsrtrurrrrrrs  ;