U
    Vc܌                     @   s  d Z ddlmZ ddlZddlZddlZddlZddlZddl	Z	zddl
ZddlZdZW n ek
rp   dZY nX zddlZdZW n ek
r   dZY nX ejdk reZddlmZmZ dZi Zd	d
 Zdd ZeddgZG dd deZG dd deZG dd deZe Z dd Z!ede  er:ede  G dd deZ"dd Z#dd Zdd  Z$d!d" Z%d#d$ Z&d-d%d&Z'd'd( Z(d)d* Z)d.d+d,Z*dS )/a&  A limited reimplementation of the TensorFlow FileIO API.

The TensorFlow version wraps the C++ FileSystem API.  Here we provide a
pure Python implementation, limited to the features required for
TensorBoard.  This allows running TensorBoard without depending on
TensorFlow for file operations.
    )
namedtupleNTF)   r   )compaterrorsi   c                 C   s   d| krt d|t| < d S )N:z$Filesystem prefix cannot contain a :)
ValueError_REGISTERED_FILESYSTEMS)prefix
filesystem r   O/tmp/pip-unpacked-wheel-g8kmtpbc/tensorboard/compat/tensorflow_stub/io/gfile.pyregister_filesystemB   s    r   c                 C   s`   t | } d}| d}|dkr,| d| }t|d}|dkrHt| }|dkr\td| |S )z4Return the registered filesystem for the given file. ://r   Nz&No recognized filesystem for prefix %s)r   
as_str_anyfindr   get_get_fsspec_filesystemr   )filenamer	   indexfsr   r   r   get_filesystemH   s    

r   StatDatalengthc                   @   sn   e Zd ZdZdd Zdd Zddd	Zdd
dZdddZdd Z	dd Z
dd Zdd Zdd Zdd ZdS )LocalFileSystemz Provides local fileystem access.c                 C   s   t jt|S (Determines whether a path exists or not.)ospathexistsr   as_bytesselfr   r   r   r   r   ^   s    zLocalFileSystem.existsc                 G   s   t jj|f| S )zJoin paths with path delimiter.)r   r   joinr"   r   pathsr   r   r   r#   b   s    zLocalFileSystem.joinFNc              
   C   s   |rdnd}|rdnd}t |s8tdddt| d}|dk	rP|dd}tj|||d>}|dk	rt|| |	|}	d|
 i}
|	|
fW  5 Q R  S Q R X dS )  Reads contents of a file to a string.

        Args:
            filename: string, a path
            binary_mode: bool, read as binary if True, otherwise text
            size: int, number of bytes or characters to read, otherwise
                read all the contents of the file (from the continuation
                marker, if present).
            continue_from: An opaque value returned from a prior invocation of
                `read(...)` marking the last read position, so that reading
                may continue from there.  Otherwise read from the beginning.

        Returns:
            A tuple of `(data, continuation_token)` where `data' provides either
            bytes read from the file (if `binary_mode == true`) or the decoded
            string representation thereof (otherwise), and `continuation_token`
            is an opaque value that can be passed to the next invocation of
            `read(...) ' in order to continue from the last read position.
        rbrNutf8Not Found: opaque_offsetencoding)r   r   NotFoundErrorr   as_textr   ioopenseekreadtell)r"   r   binary_modesizecontinue_frommoder-   offsetfdatacontinuation_tokenr   r   r   r3   f   s"      

zLocalFileSystem.readc                 C   s   |  |||rdnd dS )a  Writes string file contents to a file, overwriting any existing
        contents.

        Args:
            filename: string, a path
            file_content: string, the contents
            binary_mode: bool, write as binary if True, otherwise text
        wbwN_writer"   r   file_contentr5   r   r   r   write   s    	zLocalFileSystem.writec                 C   s   |  |||rdnd dS zAppend string file contents to a file.

        Args:
            filename: string, a path
            file_content: string, the contents to append
            binary_mode: bool, write as binary if True, otherwise text
        abaNr?   rA   r   r   r   append   s    zLocalFileSystem.appendc              	   C   sR   d|krd nd}t j|||d(}d|kr0tjntj}||| W 5 Q R X d S Nbr)   r,   )r0   r1   r   r    r/   rC   )r"   r   rB   r8   r-   r:   	compatifyr   r   r   r@      s    zLocalFileSystem._writec                 C   s6   t |tr$dd tt|D S dd |D S dS )8Returns a list of files that match the given pattern(s).c                 S   s   g | ]}t |qS r   r   r   ).0matching_filenamer   r   r   
<listcomp>   s   z(LocalFileSystem.glob.<locals>.<listcomp>c                 S   s,   g | ]$}t t|D ]}t|qqS r   )py_globglobr   r    r   )rM   Zsingle_filenamerN   r   r   r   rO      s    N)
isinstancestrrP   rQ   r   r    r!   r   r   r   rQ      s    
zLocalFileSystem.globc                 C   s   t jt|S /Returns whether the path is a directory or not.)r   r   isdirr   r    )r"   dirnamer   r   r   rV      s    zLocalFileSystem.isdirc                 C   s:   |  |stdddtt|}dd |D }|S )7Returns a list of entries contained within a directory.NzCould not find directoryc                 S   s   g | ]}t |qS r   rL   )rM   itemr   r   r   rO      s     z+LocalFileSystem.listdir.<locals>.<listcomp>)rV   r   r.   r   listdirr   r   )r"   rW   entriesr   r   r   rZ      s
    
zLocalFileSystem.listdirc                 C   s   t j|dd dS )<Creates a directory and all parent/intermediate directories.Texist_okN)r   makedirs)r"   r   r   r   r   r_      s    zLocalFileSystem.makedirsc                 C   sB   zt t|j}W n" tk
r8   tdddY nX t|S ))Returns file statistics for a given path.NCould not find file)	r   statr   r    st_sizeOSErrorr   r.   r   )r"   r   Zfile_lengthr   r   r   rb      s
    zLocalFileSystem.stat)FNN)F)F)__name__
__module____qualname____doc__r   r#   r3   rC   rG   r@   rQ   rV   rZ   r_   rb   r   r   r   r   r   [   s   
'


	r   c                   @   sl   e Zd ZdZdd Zdd Zdd Zdd	 ZdddZdddZ	dd Z
dd Zdd Zdd Zdd ZdS )S3FileSystemz!Provides filesystem access to S3.c                 C   s    t stdtjdd | _d S )Nz'boto3 must be installed for S3 support.ZS3_ENDPOINT)boto3ImportErrorr   environr   _s3_endpointr"   r   r   r   __init__   s    zS3FileSystem.__init__c                 C   sR   t |}|dr$|tdd }|d}|d| }||d d }||fS )z.Split an S3-prefixed URL into bucket and path.zs3://N/   )r   r   
startswithlenr   )r"   urlidxbucketr   r   r   r   bucket_and_path   s    


zS3FileSystem.bucket_and_pathc                 C   sJ   t jd| jd}| |\}}|j||dd}|dsB|drFdS dS )	r   s3Zendpoint_urlrp   BucketPrefixZ	DelimiterContentsCommonPrefixesTF)rj   clientrm   rw   list_objectsr   )r"   r   r   rv   r   r(   r   r   r   r      s    zS3FileSystem.existsc                 G   s   d |f| S )Join paths with a slash.rp   )r#   r$   r   r   r   r#      s    zS3FileSystem.joinFNc              
   C   s|  t jd| jd}| |\}}i }d}	|dk	r:|dd}	d}
|dk	rN|	| }
|	dks^|
dkrnd|	|
|d< z |||jf |d	  }W n tj	j
k
rF } z|jd
 d dkr4|dk	rt jd| jd}|j||d}|d }t||	| }
|	|
krd}n,d|	|
|d< |||jf |d	  }n W 5 d}~X Y nX d|	t| i}|rjt||fS |d|fS dS )r&   rx   ry   r   NZbyte_offsetr   zbytes={}-{}ZRangeBodyErrorCode)Z416ZInvalidRanger{   KeyContentLength    utf-8)rj   resourcerm   rw   r   formatZObjectr3   botocore
exceptionsClientErrorresponser   head_objectminrs   bytesdecode)r"   r   r5   r6   r7   rx   rv   r   argsr9   Zendpointstreamexcr   objcontent_lengthr<   r   r   r   r3      s:     
zS3FileSystem.readc                 C   sT   t jd| jd}| |\}}|r6t|ts@tdn
t|}|j	|||d dS )Writes string file contents to a file.

        Args:
            filename: string, a path
            file_content: string, the contents
            binary_mode: bool, write as binary if True, otherwise text
        rx   ry   zFile content type must be bytesr   r{   r   N)
rj   r   rm   rw   rR   r   	TypeErrorr   r    
put_object)r"   r   rB   r5   r   rv   r   r   r   r   rC   :  s    


zS3FileSystem.writec                 C   s   | d}| d}|dkr*td||t|d kr>g S |dd }tjd| jd	}| |\}}|d
}g }|j	||dD ]<}	|	
dg D ]*}
|
d t|d }|r|||  qq|S )rK   *?r   z{} not supported by compat globrq   Nrx   ry   r   )r{   r|   r}   r   )r   NotImplementedErrorr   rs   rj   r   rm   rw   get_paginatorpaginater   rG   )r"   r   Zstar_iZquest_ir   rv   r   pkeysr(   okeyr   r   r   rQ   L  s&    


zS3FileSystem.globc                 C   s\   t jd| jd}| |\}}|ds0|d7 }|j||dd}|dsT|drXdS dS )	rU   rx   ry   rp   rz   r}   r~   TF)rj   r   rm   rw   endswithr   r   )r"   rW   r   rv   r   r(   r   r   r   rV   h  s    
zS3FileSystem.isdirc           	         s   t jd| jd}| |\} |d} ds: d7  g }|j| ddD ]X}| fdd|dg D  |d	g D ]&}|d
 t	 d }|r~|
| q~qN|S )rX   rx   ry   r   rp   rz   c                 3   s"   | ]}|d  t  d V  qdS )r|   r   N)rs   )rM   r   r   r   r   	<genexpr>|  s    z'S3FileSystem.listdir.<locals>.<genexpr>r~   r}   r   N)rj   r   rm   rw   r   r   r   extendr   rs   rG   )	r"   rW   r   rv   r   r   r(   r   r   r   r   r   rZ   s  s    


zS3FileSystem.listdirc                 C   sN   |  |sJtjd| jd}| |\}}|ds:|d7 }|jd||d dS )r\   rx   ry   rp   r   r   N)r   rj   r   rm   rw   r   r   )r"   rW   r   rv   r   r   r   r   r_     s    

zS3FileSystem.makedirsc              
   C   s   t jd| jd}| |\}}z|j||d}t|d W S  tjjk
r } z(|j	d d dkrrt
ddd	n W 5 d}~X Y nX dS )
r`   rx   ry   r   r   r   r   Z404Nra   )rj   r   rm   rw   r   r   r   r   r   r   r   r.   )r"   r   r   rv   r   r   r   r   r   r   rb     s    zS3FileSystem.stat)FNN)F)re   rf   rg   rh   ro   rw   r   r#   r3   rC   rQ   rV   rZ   r_   rb   r   r   r   r   ri      s   
	
E
	ri   c                   @   s   e Zd ZdZdZdZdd Zdd Zdd	 Zed
d Z	dd Z
edd Zed&ddZed'ddZed(ddZdd Zdd Zedd Zedd Zed d! Zed"d# Zed$d% ZdS ))FSSpecFileSystema  Provides filesystem access via fsspec.

    The current gfile interface doesn't map perfectly to the fsspec interface
    leading to some notable inefficiencies.

    * Reads and writes to files cause the file to be reopened each time which
      can cause a performance hit when accessing local file systems.
    * walk doesn't use the native fsspec walk function so performance may be
      slower.

    See https://github.com/tensorflow/tensorboard/issues/5286 for more info on
    limitations.
    r   z::c                 C   s@   | | j}|d d D ]"}| j|krtd d d|qd S )Nr   zFfsspec URL must only have paths in the last chained filesystem, got {})splitCHAIN_SEPARATOR	SEPARATORr   InvalidArgumentErrorr   )r"   r   partspartr   r   r   _validate_path  s    
zFSSpecFileSystem._validate_pathc                    s    fdd}|S )Nc              
      sL   z | f||W S  t k
rF } ztd d t|W 5 d }~X Y nX d S N)FileNotFoundErrorr   r.   rS   )r"   r   kwargsefuncr   r   func_wrapper  s    z8FSSpecFileSystem._translate_errors.<locals>.func_wrapperr   )r   r   r   r   r   _translate_errors  s    z"FSSpecFileSystem._translate_errorsc                 C   s6   t |tr|d}| | tj|\}}||fS )Nr   )rR   r   r   r   fsspeccoreZ	url_to_fsr"   r   r   r   r   r   r   _fs_path  s
    


zFSSpecFileSystem._fs_pathc                 C   s   |  |\}}||S r   )r   r   r   r   r   r   r     s    zFSSpecFileSystem.existsc                 C   sT   g }|D ]@}| |rg }|r>|d r>|d |s>|| || qd|S )zA
        _join joins the paths with the given separator.
        r   r   )rr   r   rG   r#   )r"   sepr%   resultr   r   r   r   _join  s    

zFSSpecFileSystem._joinc           	      G   sh   |  | || j\}}}|| }tj|\}}t|}|rP||| j 7 }|| |j	|f|  S )r   )
r   
rpartitionr   r   r   Zsplit_protocolget_filesystem_classr   r   r   )	r"   r   r%   beforer   	last_pathchain_prefixprotocolr   r   r   r   r#     s    

zFSSpecFileSystem.joinFNc              
   C   s   |  |\}}|rdnd}|r"dnd}t|sFtdddt| |j|||dz}	|dk	r|	 s|tddd	||
dd}
|
dk	r|	|
 |	|}|	 rd|	 ini }||fW  5 Q R  S Q R X dS )	r&   r'   r(   Nr)   r*   r,   z{} is not seekabler+   )r   r   r   r.   r   r/   r1   seekabler   r   r   r2   r3   r4   )r"   r   r5   r6   r7   r   r   r8   r-   r:   r9   r;   r<   r   r   r   r3     s0      

zFSSpecFileSystem.readc                 C   s   |  |||rdnd dS )r   r=   r>   Nr?   rA   r   r   r   rC     s    	zFSSpecFileSystem.writec                 C   s   |  |||rdnd dS rD   r?   rA   r   r   r   rG   *  s    	zFSSpecFileSystem.appendc           	   	   C   s`   |  |\}}d|krd nd}|j|||d(}d|kr>tjntj}||| W 5 Q R X d S rH   )r   r1   r   r    r/   rC   )	r"   r   rB   r8   r   r   r-   r:   rJ   r   r   r   r@   5  s
    zFSSpecFileSystem._writec                 C   s4   | | j\}}}| | j\}}}|| | | S r   )r   r   r   )r"   r   r   Z	chain_sepr   r   r   _r   r   r   _get_chain_protocol_prefix<  s
    
z+FSSpecFileSystem._get_chain_protocol_prefixc                    sb   t |tr|d}|\}}||}j|krDj|krD|S |  fdd|D S )rK   r   c                    s,   g | ]$}j |ksj|kr |n | qS r   )r   r   )rM   filer	   r"   r   r   rO   U  s   z)FSSpecFileSystem.glob.<locals>.<listcomp>)rR   r   r   r   rQ   r   r   r   )r"   r   r   r   filesr   r   r   rQ   C  s    



zFSSpecFileSystem.globc                 C   s   |  |\}}||S rT   )r   rV   r"   rW   r   r   r   r   r   rV   \  s    zFSSpecFileSystem.isdirc                 C   s.   |  |\}}|j|dd}dd |D }|S )rX   F)Zdetailc                 S   s   g | ]}t j|qS r   )r   r   basename)rM   fnamer   r   r   rO   g  s     z,FSSpecFileSystem.listdir.<locals>.<listcomp>)r   rZ   )r"   rW   r   r   r   r   r   r   rZ   b  s    zFSSpecFileSystem.listdirc                 C   s   |  |\}}|j|ddS )r\   Tr]   )r   r_   r   r   r   r   r_   j  s    zFSSpecFileSystem.makedirsc                 C   s   |  |\}}t||S )r`   )r   r   r6   r   r   r   r   rb   p  s    zFSSpecFileSystem.stat)FNN)F)F)re   rf   rg   rh   r   r   r   r   r   r   r   r#   r3   rC   rG   r@   r   rQ   rV   rZ   r_   rb   r   r   r   r   r     s:   	

1





r   c                 C   s>   t sdS | tjd }|tjd }t|r6tS dS dS )z
    _get_fsspec_filesystem checks if the provided protocol is known to fsspec
    and if so returns the filesystem wrapper for it.
    Nr   )FSSPEC_ENABLED	partitionr   r   r   r   r   _FSSPEC_FILESYSTEM)r   segmentr   r   r   r   r   z  s    
r   r   rx   c                   @   sf   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdddZdd Z	dd Z
dd Zdd Zdd ZdS )GFilec                 C   s~   |dkrt d|t|| _t| j| _t| jd| _d | _	t
| _d| _d | _d | _d| _d|k| _d|k| _d| _d S )N)r(   r'   brr>   r=   Zbwz%mode {} not supported by compat GFilerG   r   FrI   r>   )r   r   r   r    r   r   r   hasattrfs_supports_appendbuff_DEFAULT_BLOCK_SIZEbuff_chunk_sizebuff_offsetr<   
write_tempwrite_startedr5   
write_modeclosed)r"   r   r8   r   r   r   ro     s     

zGFile.__init__c                 C   s   | S r   r   rn   r   r   r   	__enter__  s    zGFile.__enter__c                 G   s   |    d | _d| _d | _d S )Nr   )closer   r   r<   )r"   r   r   r   r   __exit__  s    zGFile.__exit__c                 C   s   | S r   r   rn   r   r   r   __iter__  s    zGFile.__iter__c                 C   s:   | j }tt| j|| }|  j |7  _ | j|||  S r   )r   r   rs   r   )r"   Znew_buff_offsetZold_buff_offset	read_sizer   r   r   _read_buffer_to_offset  s    zGFile._read_buffer_to_offsetNc                 C   s   | j rtdddd}| jrxt| j| jkrx|dk	rh| | j| }t||krV|S |}|t|8 }n| t| j}|dk	rt| j|nd}| j	
| j| j|| j\| _| _d| _|dk	r| |}n| t| j}|r|| n|}|S )a	  Reads contents of file to a string.

        Args:
            n: int, number of bytes or characters to read, otherwise
                read all the contents of the file

        Returns:
            Subset of the contents of the file as a string or bytes.
        NzFile not opened in read moder   )r   r   PermissionDeniedErrorr   rs   r   r   maxr   r   r3   r   r5   r<   )r"   nr   chunkr   r   r   r   r3     s8    
     z
GFile.readc                 C   s   | j stddd| jr(tddd| jrf| jsP| j| j	|| j
 d| _q| j| j	|| j
 nF| jdkr| j
rzdnd}t|| _| j
rtjntj}| j|| dS )zWrites string file contents to file, clearing contents of the file
        on first write and then appending on subsequent calls.

        Args:
            file_content: string, the contents
        NzFile not opened in write modeFile already closedTzw+bzw+)r   r   r   r   FailedPreconditionErrorr   r   r   rC   r   r5   rG   r   tempfileTemporaryFiler   r    r/   )r"   rB   r8   rJ   r   r   r   rC     s,        
zGFile.writec                 C   s   d }| j s<| d}|r.|d dks*| j s.|S | j st q| j d| j}|dkr|| |d | j }|rt|| n|}|S | t| j d | j }|r|| n|}|r|d dks| j s|S | j st qd S )Nrq   r   
)r   r3   StopIterationr   r   rs   )r"   liner   r   r   r   r   __next__  s$    
zGFile.__next__c                 C   s   |   S r   )r   rn   r   r   r   next"  s    z
GFile.nextc                 C   st   | j rtd d d| jsp| jd k	rp| j  | jd | j }|d k	rp| j	| j
|| j | jt| d S )Nr   r   )r   r   r   r   r   flushr2   r3   r   rC   r   r5   rs   )r"   r   r   r   r   r   %  s      


zGFile.flushc                 C   s2   |    | jd k	r(| j  d | _d| _d| _d S )NFT)r   r   r   r   r   rn   r   r   r   r   6  s    

zGFile.close)N)re   rf   rg   ro   r   r   r   r   r3   rC   r   r   r   r   r   r   r   r   r     s   
-"r   c                 C   s   t | | S )aK  Determines whether a path exists or not.

    Args:
      filename: string, a path

    Returns:
      True if the path exists, whether its a file or a directory.
      False if the path does not exist and there are no filesystem errors.

    Raises:
      errors.OpError: Propagates any errors reported by the FileSystem API.
    )r   r   r   r   r   r   r   ?  s    r   c                 C   s   t | | S )a?  Returns a list of files that match the given pattern(s).

    Args:
      filename: string or iterable of strings. The glob pattern(s).

    Returns:
      A list of strings containing filenames that match the given pattern(s).

    Raises:
      errors.OpError: If there are filesystem / directory listing errors.
    )r   rQ   r   r   r   r   rQ   O  s    rQ   c                 C   s   t | | S )zReturns whether the path is a directory or not.

    Args:
      dirname: string, path to a potential directory

    Returns:
      True, if the path is a directory; False otherwise
    )r   rV   rW   r   r   r   rV   ^  s    	rV   c                 C   s   t | | S )aY  Returns a list of entries contained within a directory.

    The list is in arbitrary order. It does not contain the special entries "."
    and "..".

    Args:
      dirname: string, path to a directory

    Returns:
      [filename1, filename2, ... filenameN] as strings

    Raises:
      errors.NotFoundError if directory doesn't exist
    )r   rZ   r   r   r   r   rZ   j  s    rZ   c                 C   s   t | | S )zCreates a directory and all parent/intermediate directories.

    It succeeds if path already exists and is writable.

    Args:
      path: string, name of the directory to be created
    )r   r_   r   r   r   r   r_   |  s    r_   c              
   c   s   t | } t| }zt| }W n< tjk
rZ } z|r@|| n
W Y dS W 5 d}~X Y nX g }g }|D ]4}|| t |}	t|	r|| qh|| qh| ||f}
|r|
V  |D ]0}|| t |}t	|||dD ]
}|V  qq|s|
V  dS )aq  Recursive directory tree generator for directories.

    Args:
      top: string, a Directory name
      topdown: bool, Traverse pre order if True, post order if False.
      onerror: optional handler for errors. Should be a function, it will be
        called with the error as argument. Rethrowing the error aborts the walk.

    Errors that happen while listing directories are ignored.

    Yields:
      Each yield is a 3-tuple:  the pathname of a directory, followed by lists
      of all its subdirectories and leaf files.
      (dirname, [subdirname, subdirname, ...], [filename, filename, ...])
      as strings
    N)onerror)
r   r   r   rZ   r   r.   r#   rV   rG   walk)toptopdownr   r   listingerrr   subdirsrY   	full_pathheresubdirZjoined_subdirsubitemr   r   r   r     s0    



r   c                 C   s   t | | S )zReturns file statistics for a given path.

    Args:
      filename: string, path to a file

    Returns:
      FileStatistics struct that contains information about the path

    Raises:
      errors.OpError: If the operation fails.
    )r   rb   r   r   r   r   rb     s    rb   c              	   C   s,   t | dd}|t| W 5 Q R X dS )zWrites a string to a given file.

    Args:
      filename: string, path to a file
      file_content: string, contents that need to be written to the file

    Raises:
      errors.OpError: If there are errors during the operation.
    r>   r8   N)r   rC   r   r/   )r   rB   r:   r   r   r   _write_string_to_file  s    
r  c                 C   s&   |rt | dd}nt | dd}| S )a  Reads the entire contents of a file to a string.

    Args:
      filename: string, path to a file
      binary_mode: whether to open the file in binary mode or not. This changes
        the type of the object returned.

    Returns:
      contents of the file as a string or bytes.

    Raises:
      errors.OpError: Raises variety of errors that are subtypes e.g.
      `NotFoundError` etc.
    r'   r  r(   )r   r3   )r   r5   r:   r   r   r   _read_file_to_string  s    r  )TN)F)+rh   collectionsr   rQ   rP   r0   r   os.pathsysr   Zbotocore.exceptionsr   rj   Z
S3_ENABLEDrk   r   r   version_inford   FileExistsErrorZ"tensorboard.compat.tensorflow_stubr   r   r   r   r   r   r   objectr   ri   r   r   r   r   r   rV   rZ   r_   r   rb   r  r  r   r   r   r   <module>   s^   


{ I Z 1
2