U &cqF@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z m Z ddlmZddlmZddlmZddlmZdd lmZmZddlZdd lmZdd lmZd d lmZe e!Z"zjej#$dd%Z&ej#$dd%Z'e'dkr0e&dkr0ddl(Z(dZ)e"*d+e(jne"*ddZ)Wne,k rZdZ)YnXzej#$dd%Z&ej#$dd%Z'e&dkre'dkrddl-Z.e/e.dre0e.jddkst1dZ2e"*d+e.jne"*ddZ2Wne,e1fk rdZ2YnXzddl3m4Z4e4Z5Wn<e,k r\ej67e8dej69e8dddZ5YnXej69e5d Z:e8d!e:Z;e8d"e;Zd%Z?d&Z@d'ZAd(ZBdgd ggdgd gggZCd)d*ddd gd dd+ddgdddd,d-ggZDd d d d d gd d d ddgdddd d ggZEd.ZFd/ZGd0d1ZHd2d3ZId4d5ZJd6d7ZKd8d9ZLd:d;ZMdQeNeNeNd<d=d>ZOdRd?d@ZPdSdAdBZQdTeeNdCdDdEZRdUdFdGZSdVeeNdCdIdJZTGdKdLdLeUZVdMdNZWdOdPZXdS)Wz Utilities for working with the local dataset cache. This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp Copyright by the AllenNLP authors. N)contextmanager)partialwraps)sha256)Path)Optional)urlparse)ZipFile is_zipfile)FileLock)tqdm) __version__USE_TFAUTO USE_TORCH)1ONYESr)rrrTzPyTorch version {} available.z'Disabling PyTorch because USE_TF is setFrz TensorFlow version {} available.z-Disabling Tensorflow because USE_TORCH is set)_get_torch_homeZ TORCH_HOMEXDG_CACHE_HOMEz~/.cachetorchZ transformersPYTORCH_PRETRAINED_BERT_CACHEPYTORCH_TRANSFORMERS_CACHETRANSFORMERS_CACHEzpytorch_model.binz tf_model.h5z model.ckptz config.jsonzmodelcard.jsonz3https://s3.amazonaws.com/models.huggingface.co/bertzhttps://cdn.huggingface.cocCstSN)_torch_availabler#r#;/tmp/pip-unpacked-wheel-ymerj3tt/transformers/file_utils.pyis_torch_available\sr%cCstSr!) _tf_availabler#r#r#r$is_tf_available`sr'csfdd}|S)Ncs$d|jdk r|jnd|_|SN)join__doc__fndocstrr#r$docstring_decoratores z1add_start_docstrings..docstring_decoratorr#r/r0r#r.r$add_start_docstringsds r2csfdd}|S)NcsPd|jdd}d|}d}||d|jdk rD|jnd|_|S)Nz:class:`~transformers.{}`.rzH The {} forward method, overrides the :func:`__call__` special method.a@ .. note:: Although the recipe for forward pass needs to be defined within this function, one should call the :class:`Module` instance afterwards instead of this since the former takes care of running the pre and post processing steps while the latter silently ignores them. r))format __qualname__splitr*r+)r- class_nameZintroZnoter.r#r$r0ms  (z=add_start_docstrings_to_callable..docstring_decoratorr#r1r#r.r$ add_start_docstrings_to_callablels r8csfdd}|S)Ncs|jd|_|Sr()r+r*r,r.r#r$r0sz/add_end_docstrings..docstring_decoratorr#r1r#r.r$add_end_docstrings~s r9cCst|}|jdkS)N)httphttps)rscheme)url_or_filenameparsedr#r#r$ is_remote_urlsr?)model_idfilenamereturncCsD|rtnt}d|k}|r,|d|d|S|d|d|SdS)a Resolve a model identifier, and a file name, to a HF-hosted url on either S3 or Cloudfront (a Content Delivery Network, or CDN). Cloudfront is replicated over the globe so downloads are way faster for the end user (and it also lowers our bandwidth costs). However, it is more aggressively cached by default, so may not always reflect the latest changes to the underlying file (default TTL is 24 hours). In terms of client-side caching from this library, even though Cloudfront relays the ETags from S3, using one or the other (or switching from one to the other) will affect caching: cached files are not shared between the two because the cached file's name contains a hash of the url. /-N)CLOUDFRONT_DISTRIB_PREFIXS3_BUCKET_PREFIX)r@rAZuse_cdnZendpointZ legacy_formatr#r#r$ hf_bucket_urls  rGcCsV|d}t|}|}|r@|d}t|}|d|7}|drR|d7}|S)a Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can identify it as a HDF5 file (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) utf-8r3z.h5)encoder hexdigestendswith)urletagZ url_bytesZurl_hashrAZ etag_bytesZ etag_hashr#r#r$url_to_filenames   rNc Cs|dkr t}t|trt|}tj||}tj|sFtd ||d}tj|shtd |t |dd}t |}W5QRX|d}|d}||fS)z Return the url and etag (which may be ``None``) stored for `filename`. Raise ``EnvironmentError`` if `filename` or its stored metadata do not exist. Nfile {} not found.jsonrH)encodingrLrM) r isinstancerstrospathr*existsEnvironmentErrorr4openjsonload)rA cache_dir cache_path meta_path meta_filemetadatarLrMr#r#r$filename_to_urls   r`)rBc  Cs|dkr t}t|trt|}t|tr0t|}t|rPt|||||||d} n>tj|rb|} n,t |j dkrt d |nt d ||rt| st| s| Stj| \} } | ddd} tj| | } tj| rt| r|s| S| d }t|tj| d d t| t| rTt| d }|| |W5QRXn8t| r~t| }|| |nt d | W5QRX| S| S)a Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and return the path to the cached file. If it's already a local path, make sure the file exists and then return the path. Args: cache_dir: specify a cache directory to save the file to (overwrite the default cache dir). force_download: if True, re-dowload the file even if it's already cached in the cache dir. resume_download: if True, resume the download if incompletly recieved file is found. user_agent: Optional string or dict that will be appended to the user-agent on remote requests. extract_compressed_file: if True and the path point to a zip or tar file, extract the compressed file in a folder along the archive. force_extract: if True when extract_compressed_file is True and the archive was already extracted, re-extract the archive and overide the folder where it was extracted. Return: None in case of non-recoverable file (non-existent or inaccessible url + no cache on disk). Local path (string) otherwise N)r[force_downloadproxiesresume_download user_agentlocal_files_onlyr)rOz.unable to parse {} as a URL or as a local pathr3rDz -extracted.lockT) ignore_errorsrz,Archive format of {} could not be identified)rrRrrSr?get_from_cacherTrUrVrr<rWr4 ValueErrorr tarfile is_tarfiler6replacer*isdirlistdirr shutilrmtreemakedirsr extractallcloserX)r=r[rarbrcrdZextract_compressed_fileZ force_extractreZ output_path output_dirZ output_fileZoutput_extract_dir_nameZoutput_path_extracted lock_pathzip_fileZtar_filer#r#r$ cached_pathsX            rxc CsFdttjd}tr,|dtj7}trB|dtj7}t |t rn|dd dd| D7}nt |t r|d|7}d|i}|dkrd |f|d <tj|d ||d }|jd krdS|jd}|dk r|t|nd} tdd | |dtttjkd} |jddD]$} | r| t| || q| dS)Nztransformers/{}; python/{}rz ; torch/{}z; tensorflow/{}z; css|]\}}d||VqdS)z{}/{}N)r4).0kvr#r#r$ 8szhttp_get..z user-agentz bytes=%d-ZRangeT)streamrbheadersizContent-LengthBZ Downloading)unitZ unit_scaletotalinitialdescdisablei) chunk_size)r4rsysversionr6r%rr'tfrRdictr*itemsrSrequestsget status_coder~intr boolloggergetEffectiveLevelloggingNOTSET iter_contentupdatelenwritert) rL temp_filerb resume_sizerdZuar~responsecontent_lengthrprogresschunkr#r#r$http_get1s< "    r c sD|dkr t}t|trt|}tj|ddd}|s~z,tj|d||d} | jdkr^| j d}Wnt tj j fk r|YnXt||} tj|| } |dkrtj| r| Sddtt|| d D} t| d krtj|| d S|rtd dStj| r|s| S| d } t| tj| rN|sN| W5QRS|r| dtfdd}|}tjrtj}nd }nttj|dd}d }|(}td||j t!|||||dW5QRXtd|| t"|j | td| ||d}| d}t#|d}t$%||W5QRXW5QRX| S)a. Given a URL, look for the corresponding file in the local cache. If it's not there, download it. Then return the path to the cached file. Return: None in case of non-recoverable file (non-existent or inaccessible url + no cache on disk). Local path (string) otherwise NT)exist_ok)allow_redirectsrbtimeoutETagcSs$g|]}|ds|ds|qS)rPrf)rK)ryfiler#r#r$ s z"get_from_cache..z.*rzCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable model look-ups and downloads online, set 'local_files_only' to False.rfz .incompletec 3s td }|VW5QRXdS)Nza+b)rX)fZincomplete_pathr#r$_resumable_file_managers z/get_from_cache.._resumable_file_managerF)dirdeletezF%s not found in cache or force_download set to True, downloading to %s)rbrrdzstoring %s in cache at %szcreating metadata file for %s)rLrMrPw)&rrRrrSrTrrrheadrr~rrW exceptionsTimeoutrNrUr*rVfnmatchfilterrorrjr rstatst_sizertempfileNamedTemporaryFilerinfonamerrmrXrYdump)rLr[rarbZ etag_timeoutrcrdrerMrrAr\Zmatching_filesrvrZtemp_file_managerrrmetar]r^r#rr$riRsn           ric@seZdZdZdddZdS)cached_propertyz Descriptor that mimics @property but caches output in member variable. From tensorflow_datasets Built-in in functools from Python 3.8. NcCsX|dkr |S|jdkrtdd|jj}t||d}|dkrT||}t||||S)Nzunreadable attributeZ __cached_)fgetAttributeError__name__getattrsetattr)selfobjZobjtypeattrcachedr#r#r$__get__s     zcached_property.__get__)N)r __module__r5r+rr#r#r#r$rsrcstfdd}|S)Ncs&tr||StdjddS)NMethod `z` requires PyTorch.)r% ImportErrorrargskwargsfuncr#r$wrappers ztorch_required..wrapperrrrr#rr$torch_requiredsrcstfdd}|S)Ncs&tr||StdjddS)Nrz` requires TF.)r'rrrrr#r$rs ztf_required..wrapperrrr#rr$ tf_requiredsr)T)N)N)NFNFNFFF)NrN)NFNrFNF)Yr+rrYrrTrprrkr contextlibr functoolsrrhashlibrpathlibrtypingr urllib.parserzipfiler r rZfilelockr Z tqdm.autor r)r getLoggerrrenvironrupperrrrr"rr4rZ tensorflowrhasattrrAssertionErrorr&Z torch.hubrZtorch_cache_homerU expandusergetenvr*Zdefault_cache_pathrrrZ WEIGHTS_NAMEZTF2_WEIGHTS_NAMEZTF_WEIGHTS_NAMEZ CONFIG_NAMEZMODEL_CARD_NAMEZMULTIPLE_CHOICE_DUMMY_INPUTSZ DUMMY_INPUTSZ DUMMY_MASKrFrEr%r'r2r8r9r?rSrGrNr`rxrripropertyrrrr#r#r#r$s           $        ((    \ #  r