U
    &ºc˜2  ã                   @   sx   d Z ddlZddlZddlZddlZe e¡Zddd„Zddd„Z	dd	d
„Z
ddd„Zddd„Zddd„Zddd„ZdS )z$ PyTorch - TF 2.0 general utilities.é    NÚ c                 C   sÎ   |   dd¡} t dd| ¡} |   dd¡} t dd| ¡} |  d¡} | dd	… } t| d
 dkpfd| kpfd| kƒ}| d
 dksŽ| d
 dksŽ| d
 dkr–d| d
< | d
 dkrªd| d
< d | ¡} |rÆ|   |dd¡} | |fS )a@   Convert a TF 2.0 model variable name in a pytorch model weight name.

        Conventions for TF2.0 scopes -> PyTorch attribute names conversions:
            - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch)
            - '_._' is replaced by a new level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList)

        return tuple with:
            - pytorch model weight name
            - transpose: boolean indicating weither TF2.0 and PyTorch weights matrices are transposed with regards to each other
    z:0r   z/[^/]*___([^/]*)/z/\1/z_._ú/z//+é   NéÿÿÿÿÚkernelZ	emb_projsZ	out_projsZ
embeddingsÚgammaÚweightÚbetaÚbiasÚ.)ÚreplaceÚreÚsubÚsplitÚboolÚjoin)Ztf_nameÚstart_prefix_to_removeÚ	transpose© r   úJ/tmp/pip-unpacked-wheel-ymerj3tt/transformers/modeling_tf_pytorch_utils.pyÚ(convert_tf_weight_name_to_pt_weight_name   s,      ÿ ÿ
 $
r   Fc                 C   s’   zddl }ddl}W n  tk
r4   t d¡ ‚ Y nX tj |¡}t d 	|¡¡ |j
|dd}t d 	tdd	„ | ¡ D ƒƒ¡¡ t| |||d
S )ú0 Load pytorch checkpoints in a TF 2.0 model
    r   NúÃLoading a PyTorch model in TensorFlow, requires both PyTorch and TensorFlow to be installed. Please see https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.zLoading PyTorch weights from {}Úcpu)Zmap_locationz+PyTorch checkpoint contains {:,} parametersc                 s   s   | ]}|  ¡ V  qd S ©N)Znumel)Ú.0Útr   r   r   Ú	<genexpr>Z   s     z7load_pytorch_checkpoint_in_tf2_model.<locals>.<genexpr>©Ú	tf_inputsÚallow_missing_keys)Ú
tensorflowÚtorchÚImportErrorÚloggerÚerrorÚosÚpathÚabspathÚinfoÚformatÚloadÚsumÚvaluesÚ!load_pytorch_weights_in_tf2_model)Útf_modelZpytorch_checkpoint_pathr   r    Útfr"   Zpt_pathÚpt_state_dictr   r   r   Ú$load_pytorch_checkpoint_in_tf2_modelI   s$    ÿ"   ÿr2   c                 C   s   |  ¡ }t| |||dS )r   r   )Z
state_dictr.   )r/   Úpt_modelr   r    r1   r   r   r   Úload_pytorch_model_in_tf2_modela   s       ÿr4   c                    sš  z ddl }ddl}ddlm} W n  tk
r@   t d¡ ‚ Y nX |dkrPˆ j}|dk	rdˆ |dd g }g }| ¡ D ]H}	d}
d|	kr|	 	dd¡}
d	|	kr¤|	 	d	d
¡}
|
rt| 
|	¡ | 
|
¡ qtt||ƒD ]\}}
| |¡||
< qÈd}t‡ fdd„| ¡ D ƒƒs
ˆ jd }ˆ jˆ j }d}g }tt| ¡ ƒƒ}|D ] }|j}t||d\}}||krp|rbq2td |¡ƒ‚||  ¡ }|rŒt |¡}t|jƒt|jƒk r®t |¡}n$t|jƒt|jƒkrÒtj|dd}zt|jƒt|jƒksît‚W n> tk
r. } z| j|j|jf7  _|‚W 5 d}~X Y nX ||j7 }| 
||f¡ |  |¡ q2| !|¡ |dk	rvˆ |dd t "d |¡¡ t "d |¡¡ ˆ S )z0 Load pytorch state_dict in a TF 2.0 model.
    r   N)Úbackendr   F©Ztrainingr   r   r	   r
   r   c                 3   s   | ]}|  ˆ j¡V  qd S r   ©Ú
startswithÚbase_model_prefix©r   Ús©r/   r   r   r   ’   s     z4load_pytorch_weights_in_tf2_model.<locals>.<genexpr>r   ©r   z{} not found in PyTorch model©Zaxisz+Loaded {:,} parameters in the TF 2.0 model.z4Weights or buffers not loaded from PyTorch model: {})#r"   r!   Ztensorflow.python.kerasr5   r#   r$   r%   Údummy_inputsÚkeysr   ÚappendÚzipÚpopÚanyr9   Ztrainable_weightsZnon_trainable_weightsÚsetÚlistÚnamer   ÚAttributeErrorr*   Únumpyr   ÚlenÚshapeÚsqueezeÚexpand_dimsÚAssertionErrorÚargsÚsizeÚdiscardZbatch_set_valuer)   )r/   r1   r   r    r"   r0   ÚKZold_keysZnew_keysÚkeyZnew_keyZold_keyr   Zsymbolic_weightsZtf_loaded_numelZweight_value_tuplesZall_pytorch_weightsZsymbolic_weightZsw_namerG   r   ÚarrayÚer   r<   r   r.   k   s€    ÿ


 ÿ





r.   c           
      C   s¬   zddl }ddl}W n  tk
r4   t d¡ ‚ Y nX ddl}t d |¡¡ d| jj	 }t
||ƒ}|| jƒ}	|dkr||	j}|dk	r|	|dd |	j|dd	 t| |	|d
S )a   Load TF 2.0 HDF5 checkpoint in a PyTorch model
        We use HDF5 to easily do transfer learning
        (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357).
    r   NúÃLoading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.z"Loading TensorFlow weights from {}ZTFFr6   T)Zby_name©r    )r!   r"   r#   r$   r%   Útransformersr)   r*   Ú	__class__Ú__name__ÚgetattrÚconfigr?   Zload_weightsÚload_tf2_model_in_pytorch_model)
r3   Ztf_checkpoint_pathr   r    r0   r"   rX   Ztf_model_class_nameZtf_model_classr/   r   r   r   Ú$load_tf2_checkpoint_in_pytorch_modelÍ   s&    ÿ

r^   c                 C   s   |j }t| ||dS )z* Load TF 2.0 model in a pytorch model
    rW   )ÚweightsÚ!load_tf2_weights_in_pytorch_model)r3   r/   r    r_   r   r   r   r]   ð   s    r]   c                    sr  zddl }ddl}W n  tk
r4   t d¡ ‚ Y nX i }tˆ  ¡ ƒ}d}t‡ fdd„| ¡ D ƒƒsnˆ j	d }i }|D ]&}	t
|	j|d\}
}|	 ¡ |f||
< qvtt| ¡ ƒƒ}i }g }| ¡ D ]:\}}| ¡ |kræ|| ¡  ||< q¾||kr|r| |¡ q¾td	 |¡ƒ‚|| \}}|r,t |¡}t|jƒt|jƒk rNt |¡}n$t|jƒt|jƒkrrtj|dd
}zt|jƒt|jƒksŽt‚W n> tk
rÎ } z| j|j|jf7  _|‚W 5 d}~X Y nX | |¡||< | |¡|| ¡ < | |¡ q¾ˆ j|dd\}}||7 }t|ƒdkr:t d ˆ jj |¡¡ t|ƒdkr^t d ˆ jj |¡¡ t d |¡¡ ˆ S )z4 Load TF2.0 symbolic weights in a PyTorch model
    r   NrV   r   c                 3   s   | ]}|  ˆ j¡V  qd S r   r7   r:   ©r3   r   r   r     s     z4load_tf2_weights_in_pytorch_model.<locals>.<genexpr>r   r=   z{} not found in TF 2.0 modelr>   F)Ústrictz3Weights of {} not initialized from TF 2.0 model: {}z,Weights from TF 2.0 model not used in {}: {}z3Weights or buffers not loaded from TF 2.0 model: {})!r!   r"   r#   r$   r%   ÚdictZnamed_parametersrD   r@   r9   r   rG   rI   rE   rF   ÚitemsZdata_ptrrA   rH   r*   r   rJ   rK   rL   rM   rN   rO   Z
from_numpyrQ   Zload_state_dictr)   rY   rZ   )r3   Z
tf_weightsr    r0   r"   Znew_pt_params_dictZcurrent_pt_params_dictr   Ztf_weights_mapZ	tf_weightZpt_namer   Zall_tf_weightsZloaded_pt_weights_data_ptrZmissing_keys_ptZpt_weight_nameZ	pt_weightrT   rU   Zmissing_keysZunexpected_keysr   ra   r   r`   ø   sv    ÿ
 ÿ



ÿÿr`   )r   )NF)NF)NF)NF)F)F)Ú__doc__Úloggingr&   r   rI   Ú	getLoggerrZ   r$   r   r2   r4   r.   r^   r]   r`   r   r   r   r   Ú<module>   s   

,



b
#
