U &c k@s>dZddlZddlZddlZddlmZddlm Z m Z ddl m Z m Z mZmZddlmZeeZdd iZd d Zd d Zd%ddZGdddejjjZd&ddZGdddejjjZeGdddejjjZGddde Z dZ!dZ"e de!Gddde Z#Gd d!d!ejjjZ$e d"e!Gd#d$d$e Z%dS)'z TF 2.0 CTRL model.N) CTRLConfig)add_start_docstrings add_start_docstrings_to_callable)TFPreTrainedModelTFSharedEmbeddingskeras_serializable shape_list) BatchEncodingZctrlz+https://cdn.huggingface.co/ctrl-tf_model.h5cCs*dtdd|dt|}||S)Nri')nppowerfloat32)posi d_model_sizeZ angle_ratesrA/tmp/pip-unpacked-wheel-ymerj3tt/transformers/modeling_tf_ctrl.py angle_defn#s"rcCstt|ddtjft|tjddf|}t|dddddf}t|dddddf}tjtj||gddtj d}|S)Nrr rZaxisZdtype) rr ZarangenewaxissincostfcastZ concatenater)positionrZ angle_radsZsinesZcosines pos_encodingrrrpositional_encoding(s 4rc Cstj||dd}tt|dtj}|tj|}|dk rJ||d7}|dk rZ||}tjj|dd} |dk rz| |} t| |} | | fS)NT)Z transpose_brr) rmatmulrr rmathsqrtnnZsoftmax) qkvmaskattention_mask head_maskZ matmul_qkZdkZscaled_attention_logitsZattention_weightsoutputrrrscaled_dot_product_attention4s  r,cs0eZdZdfdd ZddZd ddZZS) TFMultiHeadAttentionFc stjf|||_||_||_t||j|_tjj j |dd|_ tjj j |dd|_ tjj j |dd|_ tjj j |dd|_dS)NWqnameWkWvdense)super__init__output_attentions num_headsrintdepthrkeraslayersDenser.r1r2r3)selfrr7r6kwargs __class__rrr5NszTFMultiHeadAttention.__init__cCs.t||d|j|jf}tj|ddddgdS)Nrrr rperm)rreshaper7r9 transpose)r=x batch_sizerrrsplit_into_heads\sz%TFMultiHeadAttention.split_into_headscCsV|\}}}}}}} } t|d} ||}||}||}||| }||| }||| }|dk rtj|dd\} } tj| |fdd}tj| |fdd}t| rt | drt | } nd} | dkrtj ||fdd}nd}t |||||| }tj|ddddd gd }|d}t|| d |jf}||}||f}|jrR||f}|S) NrrnumpyTNr rrArBr)r r.r1r2rHrZunstackconcatZ is_tensorhasattrboolrJstackr,rErDrr3r6)r=inputstrainingr'r&r%r( layer_pastr)r* use_cacherGZpast_keyZ past_valuepresentr+Zscaled_attentionZattnZoriginal_size_attentionoutputsrrrcall`s8           zTFMultiHeadAttention.call)F)F)__name__ __module__ __qualname__r5rHrV __classcell__rrr?rr-Msr-cCs2tjjtjjj|dddtjjj|ddgddS)NZrelu0)Z activationr02r/ffn)rr:Z Sequentialr;r<)rdffr0rrrpoint_wise_feed_forward_networks$r`cs(eZdZdfdd Zd ddZZS) TFEncoderLayer皙?ư>Fc s|tjf|t|||dd|_t||dd|_tjjj |dd|_ tjjj |dd|_ tjj ||_ tjj ||_dS)Nmulti_head_attentionr/r^ layernorm1epsilonr0 layernorm2)r4r5r-rdr`r^rr:r;LayerNormalizationrerhDropoutdropout1dropout2)r=rr7r_Zratelayer_norm_epsilonr6r>r?rrr5szTFEncoderLayer.__init__c Cs|\}}}}}}||} |j| | | |||||g|d} | d} |j| |d} || } || } || }|j||d}| |} | f| dd}|S)NrQrr)rerdrkrhr^rl)r=rPrQrFr(rRr)r*rSnormedZ attn_outputsZ attn_outputZout1Zout2Z ffn_outputrUrrrrVs   zTFEncoderLayer.call)rbrcF)F)rWrXrYr5rVrZrrr?rras racsBeZdZeZfddZddZddZddZdd d Z Z S)TFCTRLMainLayerc stjf|j|_j|_j|_j|_tj |j|_ t j jj dd|_tjjj|_fddtjD|_tjjjjdd|_dS)Nw)initializer_ranger0c s4g|],}tjjjjjjd|dqS)zh_._{}r/)ran_embdZn_headr_Z resid_pdroprmr6format).0rconfigrr s z,TFCTRLMainLayer.__init__.. layernormrf)r4r5output_hidden_statesr6rsrZn_layer num_layersrZ n_positionsrr vocab_sizerrrqrr:r;rjZ embd_pdropdropoutrangehrirmry)r=rwr>r?rvrr5s"  zTFCTRLMainLayer.__init__cCs|jSrK)rqr=rrrget_input_embeddingssz$TFCTRLMainLayer.get_input_embeddingscCstdSrKNotImplementedError)r=Znew_num_tokensrrr_resize_token_embeddingssz(TFCTRLMainLayer._resize_token_embeddingscCstdS)z~ Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} Nr)r=Zheads_to_prunerrr _prune_headsszTFCTRLMainLayer._prune_headsNTFc sHt|ttfr|d} t|dkr*|dn|}t|dkrB|dn|}t|dkrZ|dn|}t|dkrr|dn|}t|dkr|dn|}t|dkr|dn|}t|dkr|dn|}t|d kstd nt|ttfrZ|d } |d |}|d |}|d|}|d|}|d|}|d|}|d|}t|d ks^td n|} |dk r| dk r| ddddf} |dk r|ddddf}|dk r|ddddf}| dk r|dk rtdnL| dk rt | } t | d| dg} n$|dk r$t |dd} ntd|dkrLd} dgt|j }nt |ddd} |dkrt j | | d| t jdt jddf}t || ddg}|dk r|ddt jt jddf}t |t j}d|d}nd}|dk rtn dg|j}|dk rVt |dt |dg}|j|dd} | t jt |jt j9} nd} t |dt |dg}|dkr|j| dd}| d}dt jt ||fdd}|t jt |jt j9}t |j|}||| }|j|| d}| t |dg}d}d}g}tt |j |D]~\}\}}|j!rH|t ||f}||||||||g| d}|dd\}}|dkr||f}|j"r"|#|dq"|$|}t ||}|j!r||f}|f}|dkr||f}|j!r||f}|j"rD| dddgt |dddtfdd |D}||f}|S)!Nrrr rAzToo many inputs. input_idspastr)token_type_ids position_idsr* inputs_embedsrSrzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrIrg?r Z embeddingmodernrTc3s|]}t|VqdSrK)rrD)rutZattention_output_shaperr ssz'TFCTRLMainLayer.call..)% isinstancetuplelistlenAssertionErrordictr get ValueErrorr rrDrr~Zint32rZtilerrrr{rqr"r#rZlinalgZ band_partZonesZgatherrr} enumerateziprzr6appendry)r=rPrr)rrr*rrSrQr input_shapeZ past_lengthZtoken_type_embedsZseq_lenr(Z pos_embeds hidden_statesZ output_shapeZpresentsZall_hidden_statesZall_attentionsrrrRrUrTrrrrVs                  (              & zTFCTRLMainLayer.call)NNNNNNTF) rWrXrYr config_classr5rrrrVrZrrr?rrps  rpc@seZdZdZeZeZdZdS)TFCTRLPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformerN) rWrXrY__doc__rr$TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAPZpretrained_model_archive_mapZbase_model_prefixrrrrrxsra .. note:: TF 2.0 models accepts two formats as inputs: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional arguments. This second option is useful when using :obj:`tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: :obj:`model(inputs)`. If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : - a single Tensor with input_ids only and nothing else: :obj:`model(inputs_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: :obj:`model([input_ids, attention_mask])` or :obj:`model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: :obj:`model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: config (:class:`~transformers.CTRLConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. a  Args: input_ids (:obj:`Numpy array` or :obj:`tf.Tensor` of shape :obj:`(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. If `past` is used, optionally only the last `input_ids` have to be input (see `past`). Indices can be obtained using :class:`transformers.CTRLTokenizer`. See :func:`transformers.PreTrainedTokenizer.encode` and :func:`transformers.PreTrainedTokenizer.encode_plus` for details. `What are input IDs? <../glossary.html#input-ids>`__ past (:obj:`List[tf.Tensor]` of length :obj:`config.n_layers`): Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past` output below). Can be used to speed up sequential decoding. If `past` is used, the user can optionally input only the last `input_ids` (those that don't have their past given to this model) of shape :obj:`(batch_size, 1)` instead of all `input_ids` of shape :obj:`(batch_size, sequence_length)`. attention_mask (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. `What are attention masks? <../glossary.html#attention-mask>`__ token_type_ids (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` corresponds to a `sentence B` token If `past` is used, optionally only the last `token_type_ids` have to be input (see `past`). `What are token type IDs? <../glossary.html#token-type-ids>`_ position_ids (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, config.max_position_embeddings - 1]``. `What are position IDs? <../glossary.html#position-ids>`_ head_mask (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`, defaults to :obj:`None`): Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: :obj:`1` indicates the head is **not masked**, :obj:`0` indicates the head is **masked**. input_embeds (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`): Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. If `past` is used, optionally only the last `input_embeds` have to be input (see `past`). use_cache (:obj:`bool`): If `use_cache` is True, `past` key value states are returned and can be used to speed up decoding (see `past`). Defaults to `True`. training (:obj:`boolean`, `optional`, defaults to :obj:`False`): Whether to activate dropout modules (if set to :obj:`True`) during training or to de-activate them (if set to :obj:`False`) for evaluation. z^The bare CTRL Model transformer outputting raw hidden-states without any specific head on top.cs,eZdZfddZeeddZZS) TFCTRLModelcs&tj|f||t|dd|_dS)Nrr/)r4r5rprr=rwrPr>r?rrr5szTFCTRLModel.__init__cKs|j|f|}|S)a( Return: :obj:`tuple(tf.Tensor)` comprising various elements depending on the configuration (:class:`~transformers.CTRLConfig`) and inputs: last_hidden_state (:obj:`tf.Tensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the last layer of the model. past (:obj:`List[tf.Tensor]` of length :obj:`config.n_layers` with each tensor of shape :obj:`(2, batch_size, num_heads, sequence_length, embed_size_per_head)`): Contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. hidden_states (:obj:`tuple(tf.Tensor)` `optional`, returned when ``config.output_hidden_states=True``): Tuple of :obj:`tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape :obj:`(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (:obj:`tuple(tf.Tensor)`, `optional`, returned when ``config.output_attentions=True``): Tuple of :obj:`tf.Tensor` (one for each layer) of shape :obj:`(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: import tensorflow as tf from transformers import CTRLTokenizer, TFCTRLModel tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = TFCTRLModel.from_pretrained('ctrl') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple )r)r=rPr>rUrrrrVs#zTFCTRLModel.call)rWrXrYr5rCTRL_INPUTS_DOCSTRINGrVrZrrr?rrs rcs0eZdZfddZfddZddZZS) TFCTRLLMHeadc s tjf||j|_||_dSrK)r4r5r|input_embeddings)r=rwrr>r?rrr5szTFCTRLLMHead.__init__cs(|j|jfdddd|_t|dS)NzerosTbias)shapeZ initializerZ trainabler0)Z add_weightr|rr4build)r=rr?rrr szTFCTRLLMHead.buildcCs|j|dd}||j}|S)NZlinearr)rr)r=rrrrrVs zTFCTRLLMHead.call)rWrXrYr5rrVrZrrr?rrs  rz~The CTRL Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). cs<eZdZfddZddZddZeeddZZ S) TFCTRLLMHeadModelcs:tj|f||t|dd|_t||jjdd|_dS)Nrr/lm_head)r4r5rprrrqrrr?rrr5szTFCTRLLMHeadModel.__init__cCs|jjSrK)rrrrrrget_output_embeddings sz'TFCTRLLMHeadModel.get_output_embeddingscKs,|rt|dddfd}|||ddS)NrrS)rPrrS)rZ expand_dims)r=rPrr>rrrprepare_inputs_for_generation#sz/TFCTRLLMHeadModel.prepare_inputs_for_generationcKs6|j|f|}|d}||}|f|dd}|S)a  Return: :obj:`tuple(tf.Tensor)` comprising various elements depending on the configuration (:class:`~transformers.CTRLConfig`) and inputs: prediction_scores (:obj:`tf.Tensor` of shape :obj:`(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). past (:obj:`List[tf.Tensor]` of length :obj:`config.n_layers` with each tensor of shape :obj:`(2, batch_size, num_heads, sequence_length, embed_size_per_head)`): Contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. hidden_states (:obj:`tuple(tf.Tensor)`, `optional`, returned when ``config.output_hidden_states=True``): Tuple of :obj:`tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape :obj:`(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (:obj:`tuple(tf.Tensor)`, `optional`, returned when ``config.output_attentions=True``): Tuple of :obj:`tf.Tensor` (one for each layer) of shape :obj:`(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: import tensorflow as tf from transformers import CTRLTokenizer, TFCTRLLMHeadModel tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = TFCTRLLMHeadModel.from_pretrained('ctrl') input_ids = tf.constant([tokenizer.encode("Links Hello, my dog is cute", add_special_tokens=True)]) outputs = model(input_ids) loss, logits = outputs[:2] rrN)rr)r=rPr>Ztransformer_outputsrZ lm_logitsrUrrrrV*s $ zTFCTRLLMHeadModel.call) rWrXrYr5rrrrrVrZrrr?rrs  r)NN)r[)&rloggingrJr Z tensorflowrZconfiguration_ctrlrZ file_utilsrrZmodeling_tf_utilsrrrr Ztokenization_utilsr getLoggerrWloggerrrrr,r:r;ZLayerr-r`rarprZCTRL_START_DOCSTRINGrrrrrrrrs@    > $B 5,