U
    &ºc1  ã                   @   sæ   d Z ddlZddlZddlZddlZddlZddlmZm	Z	 ej
ejd ddd„Zdd„ Zd	d
„ Zdd„ Zdd„ Zdd„ Zedkrâe ¡ Zejddeddd ejddeddd ejddeddd e ¡ Zeejejejƒ dS )zConvert Reformer checkpoint.é    N)ÚReformerConfigÚReformerModelWithLMHead)Úlevelc                 C   s`   | j j|jkstd | ¡ƒ‚tj |¡| _ |d k	r\| jj|jksNtd | ¡ƒ‚tj |¡| _d S )Nz{} layer.weight does not matchz{} layer.bias does not match)ÚweightÚshapeÚAssertionErrorÚformatÚtorchÚnnÚ	ParameterÚbias)Útorch_layerr   r   © r   ú\/tmp/pip-unpacked-wheel-ymerj3tt/transformers/convert_reformer_trax_checkpoint_to_pytorch.pyÚ	set_param   s
    r   c                 C   s¦   t  | d ¡}t  | d ¡}t  | d ¡}t|jjt |¡ dd¡ ¡  	d|¡ƒ t|jj
t |¡ dd¡ ¡  	d|¡ƒ t|jjt |¡ 	d|¡ ¡  dd¡ƒ d S )Nr   é   é   éÿÿÿÿ)ÚnpÚasarrayr   Úself_attentionZ	query_keyr	   ÚtensorÚ	transposeÚ
contiguousÚviewÚvalueÚoutputÚdense)Úweightsr   Úhidden_sizeZnp_query_keyÚnp_valueÚnp_denser   r   r   Úset_layer_weights_in_torch_lsh(   s    þ ÿ ÿr"   c                 C   sÜ   t  | d ¡}t  | d ¡}t  | d ¡}t  | d ¡}t|jjt |¡ dd¡ ¡  	d|¡ƒ t|jj
t |¡ dd¡ ¡  	d|¡ƒ t|jjt |¡ dd¡ ¡  	d|¡ƒ t|jjt |¡ 	d|¡ ¡  dd¡ƒ d S )Nr   r   r   é   r   )r   r   r   r   Úqueryr	   r   r   r   r   Úkeyr   r   r   )r   r   r   Znp_queryZnp_keyr    r!   r   r   r   Ú set_layer_weights_in_torch_local:   s(     ÿ ÿ ÿ ÿr&   c                 C   s†  | d d d }t  |d ¡}t  |d ¡}t|jjt |¡t |¡ƒ | d d }t|ƒdk rpt||j|ƒ nt	||j|ƒ | d d d d }t|ƒdkr¦|d }t  |d d ¡}t  |d d ¡}	t|j
jt |¡t |	¡ƒ t  |d d ¡}
t  |d d ¡}t|j
jjt |
¡ dd¡ ¡ t |¡ƒ t  |d d ¡}t  |d d ¡}t|j
jjt |¡ dd¡ ¡ t |¡ƒ d S )Nr   r   é   r   )r   r   r   Z	attentionÚ
layer_normr	   r   Úlenr"   r&   Zfeed_forwardr   r   r   r   )r   Ztorch_blockr   Zlayer_norm_1Zlayer_norm_1_weightZlayer_norm_1_biasZattn_weightsZintermediate_weightsZlayer_norm_2_weightZlayer_norm_2_biasZinter_dense_weightZinter_dense_biasZout_dense_weightZout_dense_biasr   r   r   Úset_block_weights_in_torchO   sH      ÿ  ÿýýr*   c                 C   s–  |j }t | d ¡}t|jjt |¡ƒ t| d t	ƒr¤|jj
}tt|jƒƒD ]V}t | d | d ¡}|j| j|jksŠtd || ¡ƒ‚tj t |¡¡|j|< qL| d }t|jjƒd t|ƒksÌtdƒ‚t|jjƒD ],\}	}
|d|	 d|	d  … }t||
|ƒ qØt | d d ¡}t | d d ¡}t|jjt |¡t |¡ƒ t | d	 d ¡}t | d	 d ¡}t|jjt |¡ dd¡ ¡ t |¡ƒ d S )
Nr   r#   r   z{} emb does not matché   r'   z7HF and trax model do not have the same number of layersé   é	   )Zreformerr   r   r   Z
embeddingsÚword_embeddingsr	   r   Ú
isinstanceÚtupleÚposition_embeddingsÚranger)   r   r   r   r   r
   r   ÚencoderZlayersÚ	enumerater*   r(   Zlm_headÚdecoderr   r   )r   Ztorch_modelr   Ztorch_model_reformerr.   r1   Zemb_idxZemb_weightsZtrax_layer_weightsZ	layer_idxZlayerZblock_weightsZlayer_norm_out_weightZlayer_norm_out_biasZoutput_embed_weightsZoutput_embed_biasr   r   r   Úset_model_weights_in_torch€   sJ     ÿÿÿþýýr6   c              	   C   sx   t  |¡}td t|ƒ¡ƒ t|ƒ}t| dƒ}t |¡d }W 5 Q R X t	|||j
ƒ td |¡ƒ t | ¡ |¡ d S )Nz-Building PyTorch model from configuration: {}Úrbr   zSave PyTorch model to {})r   Zfrom_json_fileÚprintr   Ústrr   ÚopenÚpickleÚloadr6   r   r	   ÚsaveZ
state_dict)Útrax_model_pkl_pathÚconfig_fileÚpytorch_dump_pathÚconfigÚmodelÚfZmodel_weightsr   r   r   Ú"convert_trax_checkpoint_to_pytorch®   s    
rD   Ú__main__z--trax_model_pkl_pathTz'Path to the TensorFlow checkpoint path.)ÚdefaultÚtypeÚrequiredÚhelpz--config_filezmThe config json file corresponding to the pre-trained Reformer model. 
This specifies the model architecture.z--pytorch_dump_pathz!Path to the output PyTorch model.)N)Ú__doc__ÚargparseÚloggingr;   Znumpyr   r	   Ztransformersr   r   ÚbasicConfigÚINFOr   r"   r&   r*   r6   rD   Ú__name__ÚArgumentParserÚparserÚadd_argumentr9   Ú
parse_argsÚargsr>   r?   r@   r   r   r   r   Ú<module>   sL   
	1.    ÿû    ÿ