U
    Vcr                     @   sl  d Z ddlZddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ e ZdZ dZ!dZ"dZ#dZ$dZ%dZ&dddddZ'dZ(G dd de)Z*G dd de)Z+dd  Z,d!d" Z-d#d$ Z.d%d& Z/d'd( Z0d)d* Z1d+d, Z2G d-d. d.ej3Z4d/d0 Z5dS )1zThe Embedding Projector plugin.    N)wrappers)json_format)text_format)context)plugin_asset_util)Respond)tf)base_plugin)metadata)ProjectorConfig)
tb_logging   z/infoz/tensorz	/metadataz/runsz
/bookmarksz/sprite_imagez	image/bmpz	image/gifz
image/jpegz	image/png)ZbmpZgifZjpegZpngapplication/octet-streamc                   @   s(   e Zd ZdZdd Zdd Zdd ZdS )	LRUCachez;LRU cache.

    Used for storing the last used tensor.
    c                 C   s$   |dk rt d|| _t | _d S )Nr   zThe cache size must be >=1)
ValueError_sizecollectionsOrderedDict_dict)selfsize r   R/tmp/pip-unpacked-wheel-g8kmtpbc/tensorboard/plugins/projector/projector_plugin.py__init__D   s    zLRUCache.__init__c                 C   s8   z| j |}|| j |< |W S  tk
r2   Y d S X d S N)r   popKeyErrorr   keyvaluer   r   r   getJ   s    
zLRUCache.getc                 C   sb   |d krt dz| j| W n2 tk
rR   t| j| jkrN| jjdd Y nX || j|< d S )Nzvalue must be != NoneF)last)r   r   r   r   lenr   popitemr   r   r   r   setR   s    zLRUCache.setN)__name__
__module____qualname____doc__r   r    r$   r   r   r   r   r   >   s   r   c                   @   s    e Zd ZdZdd Zdd ZdS )EmbeddingMetadatazMetadata container for an embedding.

    The metadata holds different columns with values used for
    visualization (color by, label by) in the "Embeddings" tab in
    TensorBoard.
    c                 C   s   || _ g | _i | _dS )zConstructs a metadata for an embedding of the specified size.

        Args:
          num_points: Number of points in the embedding.
        N)
num_pointscolumn_namesname_to_values)r   r*   r   r   r   r   e   s    zEmbeddingMetadata.__init__c                 C   s   t |tr t |d tr tdt |tjrD|jdkrDtd|j t|| jkrhtd| jt|f || jkr~td| | j	
| || j|< dS )a  Adds a named column of metadata values.

        Args:
          column_name: Name of the column.
          column_values: 1D array/list/iterable holding the column values. Must be
              of length `num_points`. The i-th value corresponds to the i-th point.

        Raises:
          ValueError: If `column_values` is not 1D array, or of length `num_points`,
              or the `name` is already used.
        r   zS"column_values" must be a flat list, but we detected that its first entry is a listr   z6"column_values" should be of rank 1, but is of rank %dz;"column_values" should be of length %d, but is of length %dz$The column name "%s" is already usedN)
isinstancelistr   npZndarrayndimr"   r*   r,   r+   append)r   Zcolumn_nameZcolumn_valuesr   r   r   
add_columno   s2     
zEmbeddingMetadata.add_columnN)r%   r&   r'   r(   r   r2   r   r   r   r   r)   ]   s   
r)   c              
   C   s`   t jj| d<}g }|D ],}|d}|r|ttt|	d qW 5 Q R X t
j|ddS )Nr
	float32dtype)r   iogfileGFilerstripr1   r.   mapfloatsplitr/   array)fpathftensorliner   r   r   _read_tensor_tsv_file   s    
&rE   c                 C   s2   t |dkrtd|tj| dd}||S )N   zTensor must be 2D, got shape {}r6   r7   )r"   r   formatr/   ZfromfileZreshape)rA   shaperC   r   r   r   _read_tensor_binary_file   s    rI   c                 C   sL   t jjtj t jj }|| krHt jt jj t j }t jt j| |S | S r   )ospathsepr
   PLUGINS_DIRpardirabspathjoin)
assets_dirZsub_pathZtwo_parents_upr   r   r   _assets_dir_to_logdir   s
    rR   c           
   
   C   s   |D ]\}}|| krjt  }tj|tj}tjj	|rrtjj
|d}| }W 5 Q R X t|| n| | }t|}t|}	|	sq|j|	kr dS qdS )zEReturns true if the latest checkpoint has changed in any of the runs.r3   TF)r   rJ   rK   rP   r
   PROJECTOR_FILENAMEr   r9   r:   existsr;   readr   MergerR   _find_latest_checkpointmodel_checkpoint_path)
configsrun_path_pairsrun_namerQ   configconfig_fpathrB   file_contentlogdir	ckpt_pathr   r   r   _latest_checkpoints_changed   s     
ra   c                 C   sL   | j |}|sdS zt|}|dkr,t |W S  tk
rF   Y dS X dS )zParses and asserts a positive (>0) integer query parameter.

    Args:
      request: The Werkzeug Request object
      param_name: Name of the parameter.

    Returns:
      Param, or None, or -1 if parameter is not a positive integer.
    Nr   )argsr    intr   )request
param_nameparamr   r   r   _parse_positive_int_param   s    
rh   c                 C   s2   t j| } t j| s.t jt j|| S | S r   )rJ   rK   
expanduserisabsrP   dirname)rA   r]   r   r   r   _rel_to_abs_asset_path   s    rl   c                   C   s
   t jdkS )zCReturn true if we're not using the fake TF API stub implementation.Zstub)r   __version__r   r   r   r   	_using_tf   s    rn   c                   @   s   e Zd ZdZejZdd Zdd Zdd Z	dd	 Z
d
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zejjdd Zejjd d! Zejjd"d# Zejjd$d% Zejjd&d' Zejjd(d) Zejjd*d+ Zd,S )-ProjectorPluginzEmbedding projector.c                 C   sB   |j | _ |j| _i | _d| _i | _d| _tt| _d| _	d| _
dS )zInstantiates ProjectorPlugin via TensorBoard core.

        Args:
          context: A base_plugin.TBContext instance.
        NF)data_providerr_   readers
_run_paths_configsconfig_fpathsr   _TENSOR_CACHE_CAPACITYtensor_cache
_is_active!_thread_for_determining_is_active)r   r   r   r   r   r      s    
zProjectorPlugin.__init__c                 C   st   d}t | jt| jt| jt| jt| j	t
| jdt| jtj|ddt| jtj|ddt| jtj|di	S )NZtf_projector_plugin	/index.jszindex.jsz/projector_binary.htmlzprojector_binary.htmlz/projector_binary.jszprojector_binary.js)
RUNS_ROUTE_serve_runsCONFIG_ROUTE_serve_configTENSOR_ROUTE_serve_tensorMETADATA_ROUTE_serve_metadataBOOKMARKS_ROUTE_serve_bookmarksSPRITE_IMAGE_ROUTE_serve_sprite_image	functoolspartial_serve_filerJ   rK   rP   )r   Zasset_prefixr   r   r   get_plugin_apps  s:             zProjectorPlugin.get_plugin_appsc                 C   sH   | j r| jsdS | jrdS | jr&| jS tj| jdd}|| _|  dS )a	  Determines whether this plugin is active.

        This plugin is only active if any run has an embedding, and only
        when running against a local log directory.

        Returns:
          Whether any run has embedding data to show in the projector.
        FTZProjectorPluginIsActiveThread)targetname)rp   r_   rw   rx   	threadingThread_determine_is_activestart)r   Z
new_threadr   r   r   	is_active  s    	zProjectorPlugin.is_activec                 C   s   t jdddS )Nry   T)Zes_module_pathZdisable_reload)r	   ZFrontendMetadatar   r   r   r   frontend_metadata>  s    z!ProjectorPlugin.frontend_metadatac                 C   s   |    | jrd| _d| _dS )zDetermines whether the plugin is active.

        This method is run in a separate thread so that the plugin can
        offer an immediate response to whether it is active and
        determine whether it should be active in a separate thread.
        TN)_update_configsrs   rw   rx   r   r   r   r   r   D  s    z$ProjectorPlugin._determine_is_activec                    s    j r4 jr4t } fdd j j|ddD }ni }| jk}| _t j } | d jkrz|	d jf |st
 j|ri  _ |\ _ _   dS )z.Updates `self._configs` and `self._run_paths`.c                    s"   i | ]}|j tj j|j qS r   )r[   rJ   rK   rP   r_   ).0runr   r   r   
<dictcomp>U  s    z3ProjectorPlugin._update_configs.<locals>.<dictcomp> )Zexperiment_id.N)rp   r_   r   ZRequestContextZ	list_runsrr   r.   items _append_plugin_asset_directoriesr1   ra   rs   rq   _read_latest_config_filesrt   %_augment_configs_with_checkpoint_info)r   ctxZ	run_pathsZrun_paths_changedrZ   r   r   r   r   P  s*    



 zProjectorPlugin._update_configsc              
   C   s  | j  D ]\}}|jD ]}|jdr:|jd d |_|jrt|j| j| }| j	||jf}|d krzt
|}W n  tk
r   t||j}Y nX | j||jf| |js|jt|t|d g q| |}|sq
d }|jr|jd js|jd }|j| | }| D ]v\}	}
t|
dkr<q"d|	krJq"| |	|}|s|j }|	|_|r|j|_|j|_|js"|j|
 q"q
g }| j  D ]\}}|js|| q|D ]}| j |= | j|= qd S )N:0r   rF   z.OPTIMIZER_SLOT)rs   r   
embeddingstensor_nameendswithtensor_pathrl   rt   rv   r    rE   UnicodeDecodeErrorrI   tensor_shaper$   extendr"   _get_reader_for_runremoveZget_variable_to_shape_map_get_embeddingaddmetadata_pathbookmarks_pathr1   )r   r   r\   	embeddingrA   rC   readerZspecial_embeddingZvar_mapr   r   Zruns_to_remover   r   r   r   o  sr    
  
 



z5ProjectorPlugin._augment_configs_with_checkpoint_infoc              
   C   s  i }i }|D ]\}}t  }tj|tj}tjj	|rhtjj
|d}| }	W 5 Q R X t|	| d}
|jD ](}|jrr|jstj|j|_d}
 qqr|jst|}t|}|s|
sq|r||_|jrt rtjj|jd std|j q|||< |||< q||fS )zLReads and returns the projector config files in every run
        directory.r3   FT*zCheckpoint file "%s" not found)r   rJ   rK   rP   r
   rS   r   r9   r:   rT   r;   rU   r   rV   r   r   r   basenamerX   rR   rW   rn   globloggerwarning)r   rZ   rY   rt   r[   rQ   r\   r]   rB   r^   Zhas_tensor_filesr   r_   r`   r   r   r   r     sN    

z)ProjectorPlugin._read_latest_config_filesc                 C   sr   || j kr| j | S | j| }d }|jrdt rdztj|j}W n" tk
rb   t	d|j Y nX || j |< |S )NzFailed reading "%s")
rq   rs   rX   rn   r   trainZload_checkpoint	Exceptionr   r   )r   r   r\   r   r   r   r   r     s    


 

z#ProjectorPlugin._get_reader_for_runc                 C   s   |  ||}|r|jS d S r   )r   r   r   r   r\   embedding_infor   r   r   _get_metadata_file_for_tensor  s    z-ProjectorPlugin._get_metadata_file_for_tensorc                 C   s   |  ||}|r|jS d S r   )r   r   r   r   r   r   _get_bookmarks_file_for_tensor  s    z.ProjectorPlugin._get_bookmarks_file_for_tensorc                 C   s   d|kr|d S |S d S )N:r   r   )r   r   r   r   r   _canonical_tensor_name  s    z&ProjectorPlugin._canonical_tensor_namec                 C   s8   |j s
d S |j D ]"}| |j| |kr|  S qd S r   )r   r   r   )r   r   r\   infor   r   r   r     s    

zProjectorPlugin._get_embeddingc           	      C   sp   g }t j}|D ]R\}}t||}t j|kr.qtj| j| t j	|}|tj
|f}|| q|| d S r   )r
   ZPLUGIN_ASSETS_NAMEr   Z
ListAssetsrS   rJ   rK   rP   rr   rM   rO   r1   r   )	r   rZ   extraZplugin_assets_namer   r_   ZassetsrQ   Zassets_path_pairr   r   r   r     s    
  z0ProjectorPlugin._append_plugin_asset_directoriesc              
   C   sX   t jt jt|}t|d.}t|d }t||	 |dW  5 Q R  S Q R X dS )zReturns a resource file.rbr   )content_typeN)
rJ   rK   rP   rk   __file__open	mimetypes
guess_typer   rU   )r   	file_pathre   Zres_path	read_filemimetyper   r   r   r     s    zProjectorPlugin._serve_filec                 C   s   |    t|t| j dS )z,Returns a list of runs that have embeddings.application/json)r   r   r.   rs   keys)r   re   r   r   r   r{   !  s    zProjectorPlugin._serve_runsc                 C   sb   |j d}|d kr"t|dddS |   | j|}|d krPt|d| ddS t|t|dS )Nr   !query parameter "run" is required
text/plain  Unknown run: "%s"r   )rc   r    r   r   rs   r   ZMessageToJson)r   re   r   r\   r   r   r   r}   '  s,            zProjectorPlugin._serve_configc              	   C   s  |j d}|d kr"t|dddS |j d}|d krDt|dddS t|d}|dkrdt|d	ddS |   | j|}|d krt|d
| ddS | ||}|st|d|| j| f ddS t|| j| }t	j
j|rt	j
j|rt|d| ddS d}t	j
j|d^}g }	|D ]N}
|	|
 t|	dkrHd|	d krHd}|rt|	|| kr qjqW 5 Q R X t|d|	dS )Nr   r   r   r   r   "query parameter "name" is requirednum_rowsrb   ,query parameter num_rows must be integer > 0r   z>No metadata file found for tensor "%s" in the config file "%s" "%s" not found, or is not a filer   r3   r   r5   r   )rc   r    r   rh   r   rs   r   rt   rl   r   r9   r:   rT   isdirr;   r1   r"   rP   )r   re   r   r   r   r\   rA   Znum_header_rowsrB   linesrD   r   r   r   r   8  sz          
   
zProjectorPlugin._serve_metadatac              
   C   s  |j d}|d kr"t|dddS |j d}|d krDt|dddS t|d}|dkrdt|d	ddS |   | j|}|d krt|d
| ddS | j||f}|d kr| ||}|r*|jr*t	|j| j
| }tjj|st|d| ddS zt|}W n" tk
r&   t||j}Y nX n| |}	|	rF|	|s^t|d||jf ddS z|	|}W n> tjjk
r }
 zt|t|
dd W Y S d }
~
X Y nX | j||f| |r|d | }|jdkr|jddd}| }t||dS )Nr   r   r   r   r   r   r   rb   r   r   zTensor file "%s" does not existz,Tensor "%s" not found in checkpoint dir "%s"r6   F)r8   copyr   )rc   r    r   rh   r   rs   rv   r   r   rl   rt   r   r9   r:   rT   rE   r   rI   r   r   Z
has_tensorrX   Z
get_tensorerrorsZInvalidArgumentErrorstrr$   r8   Zastypetobytes)r   re   r   r   r   r\   rC   r   rA   r   eZ
data_bytesr   r   r   r   t  s          
   
  
(zProjectorPlugin._serve_tensorc              	   C   s  |j d}|st|dddS |j d}|d kr@t|dddS |   | j|}|d krnt|d| ddS | ||}|st|d|| j| f ddS t|| j| }tj	j
|rtj	j
|rt|d	| ddS d }tj	j
|d
}| }W 5 Q R X t||dS )Nr   r   r   r   r   r   r   z?No bookmarks file found for tensor "%s" in the config file "%s"r   r   r   )rc   r    r   r   rs   r   rt   rl   r   r9   r:   rT   r   r;   rU   )r   re   r   r   r\   rA   Zbookmarks_jsonrB   r   r   r   r     s^             z ProjectorPlugin._serve_bookmarksc                 C   s4  |j d}|st|dddS |j d}|d kr@t|dddS |   | j|}|d krnt|d| ddS | ||}|r|jjst|d|| j| f ddS t	j
|jj}t|| j| }tjj|rtjj|rt|d	| ddS tjj|d
}| }|  td |}	t|	t}
t|||
S )Nr   r   r   r   r   r   r   zBNo sprite image file found for tensor "%s" in the config file "%s"z#"%s" does not exist or is directoryr   )rc   r    r   r   rs   r   ZspriteZ
image_pathrt   rJ   rK   ri   rl   r   r9   r:   rT   r   r;   rU   closeimghdrwhat_IMGHDR_TO_MIMETYPE_DEFAULT_IMAGE_MIMETYPE)r   re   r   r   r\   r   rA   rB   Zencoded_image_stringZ
image_typeZ	mime_typer   r   r   r     sd             z#ProjectorPlugin._serve_sprite_imageN)r%   r&   r'   r(   r
   ZPLUGIN_NAMEZplugin_namer   r   r   r   r   r   r   r   r   r   r   r   r   r   r   RequestZapplicationr   r{   r}   r   r   r   r   r   r   r   r   ro      s<   "B.




;
J
*ro   c                 C   sX   t  s
d S z.tj| }|s4tjtj| tj}|W S  tjj	k
rR   Y d S X d S r   )
rn   r   r   Zlatest_checkpointrJ   rK   rP   rN   r   ZNotFoundError)Zdir_pathr`   r   r   r   rW     s    rW   )6r(   r   r   r   r   rJ   r   Znumpyr/   Zwerkzeugr   Zgoogle.protobufr   r   Ztensorboardr   Z$tensorboard.backend.event_processingr   Ztensorboard.backend.http_utilr   Ztensorboard.compatr   Ztensorboard.pluginsr	   Ztensorboard.plugins.projectorr
   Z2tensorboard.plugins.projector.projector_config_pb2r   Ztensorboard.utilr   Z
get_loggerr   ru   r|   r~   r   rz   r   r   r   r   objectr   r)   rE   rI   rR   ra   rh   rl   rn   ZTBPluginro   rW   r   r   r   r   <module>   s^   :
    5