U
    &ºcb  ã                   @   st   d dl Z ddlmZ ddlmZ e  e¡ZdZdZ	ddd	d
gZ
ddiZG dd„ deƒZdgZdZG dd„ deƒZdS )é    Né   )ÚRobertaTokenizer)ÚXLMRobertaTokenizerzLhttps://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.jsonzLhttps://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-merges.txtz
bart-largezbart-large-mnlizbart-large-cnnzbart-large-xsumÚ
vocab_filezsentence.bpe.modelc                   @   s8   e Zd Zdd„ eD ƒZdd„ eD ƒdd„ eD ƒdœZdS )ÚBartTokenizerc                 C   s   i | ]
}|d “qS ©i   © ©Ú.0Úmr   r   úB/tmp/pip-unpacked-wheel-ymerj3tt/transformers/tokenization_bart.pyÚ
<dictcomp>#   s      zBartTokenizer.<dictcomp>c                 C   s   i | ]
}|t “qS r   )Ú	vocab_urlr	   r   r   r   r   %   s      c                 C   s   i | ]
}|t “qS r   )Ú
merges_urlr	   r   r   r   r   &   s      )r   Zmerges_fileN)Ú__name__Ú
__module__Ú__qualname__Ú_all_bart_modelsÚmax_model_input_sizesÚpretrained_vocab_files_mapr   r   r   r   r   !   s   þr   zmbart-large-en-rozahttps://s3.amazonaws.com/models.huggingface.co/bert/facebook/mbart-large-en-ro/sentence.bpe.modelc                   @   s0   e Zd ZeZdd„ eD ƒZddd„ eD ƒiZdS )ÚMBartTokenizerc                 C   s   i | ]
}|d “qS r   r   r	   r   r   r   r   0   s      zMBartTokenizer.<dictcomp>r   c                 C   s   i | ]
}|t “qS r   )ÚSPM_URLr	   r   r   r   r   1   s      N)r   r   r   ÚVOCAB_FILES_NAMESZvocab_files_namesÚ_all_mbart_modelsr   r   r   r   r   r   r   .   s   r   )ÚloggingZtokenization_robertar   Ztokenization_xlm_robertar   Ú	getLoggerr   Úloggerr   r   r   r   r   r   r   r   r   r   r   r   Ú<module>   s   
	