U
    Vc|J                     @   s   d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ddl
mZ ddl
mZ e ZG dd	 d	e	jZd
d Zdd Zdd Zdd Zdd Zdd Zdd ZdS )z;Bridge from event multiplexer storage to generic data APIs.    N)errors)summary_pb2)provider)
tb_logging)tensor_utilc                   @   s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	d,ddZ
d-ddZd.ddZd/ddddZd0dddddZd1ddddZd2dddddZd d! Zd"d# Zd$d% Zd3ddd&d'Zd4dddd(d)Zd5d*d+ZdS )6MultiplexerDataProviderc                 C   s   || _ || _dS )a6  Trivial initializer.

        Args:
          multiplexer: A `plugin_event_multiplexer.EventMultiplexer` (note:
            not a boring old `event_multiplexer.EventMultiplexer`).
          logdir: The log directory from which data is being read. Only used
            cosmetically. Should be a `str`.
        N)_multiplexer_logdir)selfZmultiplexerZlogdir r   V/tmp/pip-unpacked-wheel-g8kmtpbc/tensorboard/backend/event_processing/data_provider.py__init__    s    	z MultiplexerDataProvider.__init__c                 C   s
   d| j  S )Nz"MultiplexerDataProvider(logdir=%r))r	   r
   r   r   r   __str__,   s    zMultiplexerDataProvider.__str__c                 C   s    t |jdkrtd|f d S )NZRequestContextz%ctx must be a RequestContext; got: %r)type__name__	TypeError)r
   ctxr   r   r   _validate_context/   s    z)MultiplexerDataProvider._validate_contextc                 C   s$   t |ts tdtt||f d S )Nz(experiment_id must be %r, but got %r: %r)
isinstancestrr   r   )r
   experiment_idr   r   r   _validate_experiment_id3   s    
z/MultiplexerDataProvider._validate_experiment_idc                 C   s6   |d krt dt|trd S t dt||f d S )Nz#`downsample` required but not givenz+`downsample` must be an int, but got %r: %r)r   r   intr   )r
   
downsampler   r   r   _validate_downsample=   s    

z,MultiplexerDataProvider._validate_downsamplec                 C   s8   |j }|d k	r||krdS |j}|d k	r4||kr4dS dS )NFTrunstags)r
   run_tag_filterruntagr   r   r   r   r   _test_run_tagG   s    z%MultiplexerDataProvider._test_run_tagc              
   C   s<   z| j |W S  tk
r6 } z
W Y d S d }~X Y nX d S N)r   ZFirstEventTimestamp
ValueError)r
   run_nameer   r   r   _get_first_event_timestampP   s    z2MultiplexerDataProvider._get_first_event_timestampNc                C   s"   |  | | | tj| jdS )N)Zdata_location)r   r   r   ZExperimentMetadatar	   r
   r   r   r   r   r   experiment_metadataV   s    

z+MultiplexerDataProvider.experiment_metadatac                C   s   |  | | | | j S r#   )r   r   r   ZActivePluginsr(   r   r   r   list_plugins[   s    

z$MultiplexerDataProvider.list_pluginsc                   s,     |  |  fdd j D S )Nc                    s"   g | ]}t j|| |d qS ))Zrun_idr%   
start_time)r   ZRunr'   ).0r    r   r   r   
<listcomp>h   s   z5MultiplexerDataProvider.list_runs.<locals>.<listcomp>)r   r   r   ZRunsr(   r   r   r   	list_runse   s
    


z!MultiplexerDataProvider.list_runs)r   c                C   s2   |  | | | | ||tj}| tj|S r#   )r   r   _indexr   DATA_CLASS_SCALAR_listr   ZScalarTimeSeriesr
   r   r   plugin_namer   indexr   r   r   list_scalarsq   s    

  z$MultiplexerDataProvider.list_scalars)r   r   c                C   s<   |  | | | | | | ||tj}| t||S r#   )r   r   r   r/   r   r0   _read_convert_scalar_eventr
   r   r   r3   r   r   r4   r   r   r   read_scalars{   s    	


  z$MultiplexerDataProvider.read_scalarsc                C   s2   |  | | | | ||tj}| tj|S r#   )r   r   r/   r   DATA_CLASS_TENSORr1   r   ZTensorTimeSeriesr2   r   r   r   list_tensors   s    

  z$MultiplexerDataProvider.list_tensorsc                C   s<   |  | | | | | | ||tj}| t||S r#   )r   r   r   r/   r   r:   r6   _convert_tensor_eventr8   r   r   r   read_tensors   s    	


  z$MultiplexerDataProvider.read_tensorsc                 C   s  |dkrt jddd}|j}|j}|rt|dkr|rt|dkr|\}|\}z| j||}W n tk
rx   i  Y S X |||ii}	n
| j }	i }
|		 D ]l\}}|dk	r||krqi }|	 D ]D\}}|dk	r||krq|j
|krq|jj|krq||
|< |||< qq|
S )a  List time series and metadata matching the given filters.

        This is like `_list`, but doesn't traverse `Tensors(...)` to
        compute metadata that's not always needed.

        Args:
          plugin_name: A string plugin name filter (required).
          run_tag_filter: An `provider.RunTagFilter`, or `None`.
          data_class_filter: A `summary_pb2.DataClass` filter (required).

        Returns:
          A nested dict `d` such that `d[run][tag]` is a
          `SummaryMetadata` proto.
        Nr      )r   ZRunTagFilterr   r   lenr   SummaryMetadataKeyErrorZAllSummaryMetadataitems
data_classplugin_datar3   )r
   r3   r   Zdata_class_filterr   r   r    r!   metadataZall_metadataresulttag_to_metadataresult_for_runr   r   r   r/      s8     


zMultiplexerDataProvider._indexc              	   C   s   i }|  D ]\}}i }|||< |  D ]\}}d}	d}
| j||D ]4}|	dks\|	|jk rb|j}	|
dkst|
|jk rF|j}
qF| j||}||	|
|jj|j|j	d||< q(q|S )ac  Helper to list scalar or tensor time series.

        Args:
          construct_time_series: `ScalarTimeSeries` or `TensorTimeSeries`.
          index: The result of `self._index(...)`.

        Returns:
          A list of objects of type given by `construct_time_series`,
          suitable to be returned from `list_scalars` or `list_tensors`.
        N)max_stepmax_wall_timeplugin_contentdescriptiondisplay_name)
rB   r   Tensorsstep	wall_timer@   rD   contentsummary_descriptionrM   )r
   Zconstruct_time_seriesr4   rF   r    rG   rH   r!   summary_metadatarI   rJ   eventr   r   r   r1      s*    zMultiplexerDataProvider._listc                    sf   i }|  D ]T\}}i }|||< |  D ]6\}}	| j||}
 fdd|
D }t||||< q(q|S )a  Helper to read scalar or tensor data from the multiplexer.

        Args:
          convert_event: Takes `plugin_event_accumulator.TensorEvent` to
            either `provider.ScalarDatum` or `provider.TensorDatum`.
          index: The result of `self._index(...)`.
          downsample: Non-negative `int`; how many samples to return per
            time series.

        Returns:
          A dict of dicts of values returned by `convert_event` calls,
          suitable to be returned from `read_scalars` or `read_tensors`.
        c                    s   g | ]} |qS r   r   r,   r&   convert_eventr   r   r-     s     z1MultiplexerDataProvider._read.<locals>.<listcomp>)rB   r   rN   _downsample)r
   rW   r4   r   rF   r    Ztags_for_runrH   r!   rE   eventsdatar   rV   r   r6      s    zMultiplexerDataProvider._readc             
   C   s   |  | | | | ||tj}i }| D ]\}}i }	|	||< | D ]\}
}d }d }d }| j||
D ]R}|d ks||jk r|j}|d ks||j	k r|j	}t
|j}|d ks||krn|}qntj||||jj|j|jd|	|
< qLq0|S )N)rI   rJ   
max_lengthrK   rL   rM   )r   r   r/   r   DATA_CLASS_BLOB_SEQUENCErB   r   rN   rO   rP   _tensor_sizetensor_protor   ZBlobSequenceTimeSeriesrD   rQ   rR   rM   )r
   r   r   r3   r   r4   rF   r    rG   rH   r!   rE   rI   rJ   r[   rT   lengthr   r   r   list_blob_sequences  s@    

  
z+MultiplexerDataProvider.list_blob_sequencesc             	   C   s   |  | | | | | | ||tj}i }| D ]\}}	i }
|
||< |	D ]f}| j||}i }|D ]&}|j	|kr|qlt
|||||||j	< qldd t| D }t|||
|< qRq:|S )Nc                 S   s   g | ]\}}|qS r   r   )r,   rO   Zdatumr   r   r   r-   O  s     z?MultiplexerDataProvider.read_blob_sequences.<locals>.<listcomp>)r   r   r   r/   r   r\   rB   r   rN   rO   _convert_blob_sequence_eventsortedrX   )r
   r   r   r3   r   r   r4   rF   r    r   rH   r!   rY   Zdata_by_steprT   rZ   r   r   r   read_blob_sequences3  s8    	


  
    z+MultiplexerDataProvider.read_blob_sequencesc                   s   |  | t|\}}}} }| j||}|jtjkrBt|| j	||}	t
 fdd|	D d }
|
s~td| f t|
j}|| S )Nc                 3   s   | ]}|j  kr|V  qd S r#   rO   rU   rd   r   r   	<genexpr>c  s     
 z4MultiplexerDataProvider.read_blob.<locals>.<genexpr>z%s: no such step %r)r   _decode_blob_keyr   r@   rC   r   r\   r   ZNotFoundErrorrN   nextr   make_ndarrayr^   )r
   r   Zblob_keyZunused_experiment_idr3   r    r!   r4   rS   Ztensor_eventsZmatching_stepZtensorr   rd   r   	read_blobS  s$    

z!MultiplexerDataProvider.read_blob)N)N)N)N)N)N)N)N)N)N)r   
__module____qualname__r   r   r   r   r   r"   r'   r)   r*   r.   r5   r9   r;   r=   r/   r1   r6   r`   rc   ri   r   r   r   r   r      sR   

	



    1! $  r   c           	      C   s>   t j| |||||fdd}|d}t|}|ddS )a  Generate a blob key: a short, URL-safe string identifying a blob.

    A blob can be located using a set of integer and string fields; here we
    serialize these to allow passing the data through a URL.  Specifically, we
    1) construct a tuple of the arguments in order; 2) represent that as an
    ascii-encoded JSON string (without whitespace); and 3) take the URL-safe
    base64 encoding of that, with no padding.  For example:

        1)  Tuple: ("some_id", "graphs", "train", "graph_def", 2, 0)
        2)   JSON: ["some_id","graphs","train","graph_def",2,0]
        3) base64: WyJzb21lX2lkIiwiZ3JhcGhzIiwidHJhaW4iLCJncmFwaF9kZWYiLDIsMF0K

    Args:
      experiment_id: a string ID identifying an experiment.
      plugin_name: string
      run: string
      tag: string
      step: int
      index: int

    Returns:
      A URL-safe base64-encoded string representing the provided arguments.
    ),:)
separatorsascii=)jsondumpsencodebase64urlsafe_b64encodedecoderstrip)	r   r3   r    r!   rO   r4   stringifiedZ
bytesifiedencodedr   r   r   _encode_blob_keyk  s    

rz   c           	      C   s>   t | d }|d}t|\}}}}}}||||||fS )a'  Decode a blob key produced by `_encode_blob_key` into component fields.

    Args:
      key: a blob key, as generated by `_encode_blob_key`.

    Returns:
      A tuple of `(experiment_id, plugin_name, run, tag, step, index)`, with types
      matching the arguments of `_encode_blob_key`.
    z==ro   )rt   urlsafe_b64decoderv   rq   loads)	keydecodedrx   r   r3   r    r!   rO   r4   r   r   r   rf     s    

rf   c                 C   s    t j| j| jt| j dS )zHelper for `read_scalars`.)rO   rP   value)r   ZScalarDatumrO   rP   r   rh   r^   itemrT   r   r   r   r7     s
    r7   c                 C   s   t j| j| jt| jdS )zHelper for `read_tensors`.)rO   rP   Znumpy)r   ZTensorDatumrO   rP   r   rh   r^   r   r   r   r   r<     s
    
r<   c                    s@   t  j}t fddt|D }tj j j|dS )z!Helper for `read_blob_sequences`.c              
   3   s(   | ] }t t j|V  qd S r#   )r   ZBlobReferencerz   rO   )r,   idxrT   r   r3   r    r!   r   r   re     s   z/_convert_blob_sequence_event.<locals>.<genexpr>)rP   rO   values)r]   r^   tupleranger   ZBlobSequenceDatumrP   rO   )r   r3   r    r!   rT   Z	num_blobsr   r   r   r   ra     s    
ra   c                 C   s    d}| j jD ]}||j9 }q|S )zCompute the number of elements in a tensor.

    This does not deserialize the full tensor contents.

    Args:
      tensor_proto: A `tensorboard.compat.proto.tensor_pb2.TensorProto`.

    Returns:
      A non-negative `int`.
    r>   )Ztensor_shapedimsize)r^   rF   r   r   r   r   r]     s    r]   c                    sn   |t  krt S |dkr g S tdtt  d |d }|  |t  d g7 } fdd|D S )av  Downsample `xs` to at most `k` elements.

    If `k` is larger than `xs`, then the contents of `xs` itself will be
    returned. If `k` is smaller than `xs`, the last element of `xs` will
    always be included (unless `k` is `0`) and the preceding elements
    will be selected uniformly at random.

    This differs from `random.sample` in that it returns a subsequence
    (i.e., order is preserved) and that it permits `k > len(xs)`.

    The random number generator will always be `random.Random(0)`, so
    this function is deterministic (within a Python process).

    Args:
      xs: A sequence (`collections.abc.Sequence`).
      k: A non-negative integer.

    Returns:
      A new list whose elements are a subsequence of `xs` of length
      `min(k, len(xs))` and that is guaranteed to include the last
      element of `xs`, uniformly selected among such subsequences.
    r   r>   c                    s   g | ]} | qS r   r   )r,   ixsr   r   r-     s     z_downsample.<locals>.<listcomp>)r?   listrandomRandomsampler   sort)r   kindicesr   r   r   rX     s    "rX   )__doc__rt   rq   r   Ztensorboardr   Ztensorboard.compat.protor   Ztensorboard.datar   Ztensorboard.utilr   r   Z
get_loggerloggerZDataProviderr   rz   rf   r7   r<   ra   r]   rX   r   r   r   r   <module>   s&     N)		