U &cb@s dZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z ddl mZmZeeZdd iZd d Zd d Zd ddZGdddejjZddZGdddejjZGdddeZdZdZe deGdddeZe deGdddeZdS)!z PyTorch CTRL model.N)CrossEntropyLoss) CTRLConfig)add_start_docstrings add_start_docstrings_to_callable)Conv1DPreTrainedModelZctrlz?https://storage.googleapis.com/sf-ctrl/pytorch/seqlen256_v1.bincCs$dtdd|d|}||S)Nri')torchpow)posi d_model_sizeZ angle_ratesr>/tmp/pip-unpacked-wheel-ymerj3tt/transformers/modeling_ctrl.py angle_defn$srcCszttj||ddtj||dd|}t|dddddf}t|dddddf}tj||gdd}|S)Ndtyperrr Zdim)rr arange unsqueezesincoscat)positionrrZ angle_radsZsinesZcosines pos_encodingrrrpositional_encoding)src Cst||dddd}|jd}|t|}|dk rn|d|d} } ||| | | d| fd7}|dk r~||}tj|dd} |dk r| |} t| |} | | fS) Nrrr rr)r matmulpermuteshapenpsqrtsizeZsoftmax) qkvmaskattention_mask head_maskZ matmul_qkZdkZscaled_attention_logitsZndnsZattention_weightsoutputrrrscaled_dot_product_attention8s   r/cs0eZdZd fdd ZddZd ddZZS) MultiHeadAttentionFcspt||_||_||_t||j|_tj |||_ tj |||_ tj |||_ tj |||_ dSN)super__init__output_attentions num_headsrintdepthr nnLinearWqWkWvdense)selfrr5r4 __class__rrr3Ss zMultiHeadAttention.__init__cCs&||d|j|j}|ddddgS)Nrrr rr)reshaper5r7r")r>x batch_sizerrrsplit_into_headsasz#MultiHeadAttention.split_into_headsNc Cs|jd} ||}||}||}||| }||| }||| }|dk r|d|d} } tj| |fdd}tj| |fdd}|dkrt||f} nd} t||||||} | d ddddg}| d}| | d |j }| |} | | f}|j r ||f}|S) NrrrrTr1r rr)r#r:r;r<rDr rstackr/r"rArr=r4)r>r)r(r'r* layer_pastr+r, use_cacherCZpast_keyZ past_valuepresentr.Zscaled_attentionattnZoriginal_size_attentionoutputsrrrforwardes.         zMultiHeadAttention.forward)F)NNNF)__name__ __module__ __qualname__r3rDrK __classcell__rrr?rr0Rsr0cCs*tjtj||tjtj||Sr1)r r8Z Sequentialr9ZReLU)rdffrrrpoint_wise_feed_forward_networksrQcs(eZdZdfdd Zd ddZZS) EncoderLayer皙?Fcshtt||||_t|||_tjj|dd|_ tjj|dd|_ tj ||_ tj ||_ dS)Ngư>Zeps)r2r3r0multi_head_attentionrQffnr r8 LayerNorm layernorm1 layernorm2Dropoutdropout1dropout2)r>rr5rPZrater4r?rrr3s  zEncoderLayer.__init__Nc Csz||}|j||||||||d}|d} || } || } || } || } || } | | } | f|dd} | S)NrFr+r,rGrr)rXrUr[rYrVr\)r>rBr*rFr+r,rGnormedZ attn_outputsZ attn_outputZout1Zout2Z ffn_outputrJrrrrKs(      zEncoderLayer.forward)rSF)NNNF)rLrMrNr3rKrOrrr?rrRs rRc@s$eZdZdZeZeZdZddZ dS)CTRLPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformercCs|t|tjtjtfrR|jjjd|jj dt|tjtfrx|j dk rx|j j n&t|tj rx|j j |jj ddS)z! Initialize the weights. g)ZmeanZstdN?) isinstancer8r9 EmbeddingrZweightdataZnormal_configZinitializer_rangebiasZzero_rWZfill_)r>modulerrr _init_weightss  z!CTRLPreTrainedModel._init_weightsN) rLrMrN__doc__rZ config_class!CTRL_PRETRAINED_MODEL_ARCHIVE_MAPZpretrained_model_archive_mapZbase_model_prefixrhrrrrr_s r_am This model is a PyTorch `torch.nn.Module `_ sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config (:class:`~transformers.CTRLConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. a Args: input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. If `past` is used, optionally only the last `input_ids` have to be input (see `past`). Indices can be obtained using :class:`transformers.CTRLTokenizer`. See :func:`transformers.PreTrainedTokenizer.encode` and :func:`transformers.PreTrainedTokenizer.encode_plus` for details. `What are input IDs? <../glossary.html#input-ids>`__ past (:obj:`List[torch.FloatTensor]` of length :obj:`config.n_layers`): Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past` output below). Can be used to speed up sequential decoding. If `past` is used, the user can optionally input only the last `input_ids` (those that don't have their past given to this model) of shape :obj:`(batch_size, 1)` instead of all `input_ids` of shape :obj:`(batch_size, sequence_length)`. attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. `What are attention masks? <../glossary.html#attention-mask>`__ token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` corresponds to a `sentence B` token If `past` is used, optionally only the last `token_type_ids` have to be input (see `past`). `What are token type IDs? <../glossary.html#token-type-ids>`_ position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, config.max_position_embeddings - 1]``. `What are position IDs? <../glossary.html#position-ids>`_ head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`, defaults to :obj:`None`): Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: :obj:`1` indicates the head is **not masked**, :obj:`0` indicates the head is **masked**. input_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`): Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. If `past` is used, optionally only the last `input_embeds` have to be input (see `past`). use_cache (:obj:`bool`): If `use_cache` is True, `past` key value states are returned and can be used to speed up decoding (see `past`). Defaults to `True`. z^The bare CTRL Model transformer outputting raw hidden-states without any specific head on top.c sFeZdZfddZddZddZddZeed d d Z Z S) CTRLModelcstj|_j|_j|_j|_tj |jt j |_ t jj|_t j|_t fddtjD|_t jjjd|_|dS)Ncs&g|]}tjjjjjqSr)rRn_embdZn_headrPZ resid_pdropr4).0_rerr sz&CTRLModel.__init__..rT)r2r3output_hidden_statesr4rlrn_layerZ num_layersrZ n_positionsr floatrr8rc vocab_sizewrZZ embd_pdropdropoutZ ModuleListrangehrWZlayer_norm_epsilon layernorm init_weightsr>rer?rorr3s  zCTRLModel.__init__cCs|jSr1rur>rrrget_input_embeddingsszCTRLModel.get_input_embeddingscCs ||_dSr1r|)r>Znew_embeddingsrrrset_input_embeddings"szCTRLModel.set_input_embeddingscCs(|D]\}}|j|j|qdS)z~ Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} N)itemsrxrIZ prune_heads)r>Zheads_to_pruneZlayerZheadsrrr _prune_heads%szCTRLModel._prune_headsNTc  s|dk r\|dk r$|ddddf}|dk r@|ddddf}|dk r\|ddddf}|dk rv|dk rvtdnX|dk r|} |d| d}|jd} n,|dk r|dd} |jd} ntd|dkrd} dgt|j}n|ddd} |dkrP|dk r|jn|j} tj| | d| tj | d}| dd| d}|dk r| dkslt d|| d}| d  d }|j |j d }d |d }|||jj}|dk r|d| d}||} | t|j9} nd} |d| d}|dkr||}| d}tt|| || d  |j}|t|j9}|j|ddf |j}||| }||}| |df}d}d}g}tt|j|D]x\}\}}|jr||j|f}||||||||d}|dd \}}|dkr||f}|jr||d q||}|j|}|jrH||f}|f}|dkrb||f}|jrt||f}|jr| ddd|djddtfdd|D}||f}|S)a Return: :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.CTRLConfig`) and inputs: last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the last layer of the model. past (:obj:`List[torch.FloatTensor]` of length :obj:`config.n_layers` with each tensor of shape :obj:`(2, batch_size, num_heads, sequence_length, embed_size_per_head)`): Contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``): Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of shape :obj:`(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``): Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: from transformers import CTRLTokenizer, CTRLModel import torch tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = CTRLModel.from_pretrained('ctrl') input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple NrzDYou cannot specify both input_ids and inputs_embeds at the same timerz5You have to specify either input_ids or inputs_embedsr)rdevicez$batch_size has to be defined and > 0rr rrar rr]T)rc3s|]}|jVqdSr1)view)rmtZattention_output_shaperr sz$CTRLModel.forward..) ValueErrorr&rr#lenrxrr rlongrAssertionErrortorZ get_head_maskrerrrur$r%rZtriuZonesrrv enumerateziprqr4appendrytuple)r> input_idspastr+token_type_ids position_idsr, inputs_embedsrGZ input_shaperCZ past_lengthrZtoken_type_embedsZseq_lenr*Z pos_embeds hidden_statesZ output_shapeZpresentsZall_hidden_statesZall_attentionsr rxrFrJrHrrrrK,s1           $          " zCTRLModel.forward)NNNNNNNT) rLrMrNr3r~rrrCTRL_INPUTS_DOCSTRINGrKrOrrr?rrks rkz~The CTRL Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). c s>eZdZfddZddZddZeed d d ZZ S) CTRLLMHeadModelcs8t|t||_tj|j|jdd|_| dS)NT)rf) r2r3rkr`r8r9rlrtlm_headrzr{r?rrr3s  zCTRLLMHeadModel.__init__cCs|jSr1)rr}rrrget_output_embeddingssz%CTRLLMHeadModel.get_output_embeddingscKs*|r|dddfd}|||ddS)NrrG)rrrG)r)r>rrkwargsrrrprepare_inputs_for_generationsz-CTRLLMHeadModel.prepare_inputs_for_generationNTc  Cs|j|||||||| d} | d} || } | f| dd} |dk r| dddddf}|dddf}t}||d|d|d}|f| } | S)aY labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` Indices are selected in ``[-100, 0, ..., config.vocab_size]`` All labels set to ``-100`` are ignored (masked), the loss is only computed for labels in ``[0, ..., config.vocab_size]`` Return: :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.CTRLConfig`) and inputs: loss (:obj:`torch.FloatTensor` of shape `(1,)`, `optional`, returned when ``labels`` is provided) Language modeling loss. prediction_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). past (:obj:`List[torch.FloatTensor]` of length :obj:`config.n_layers` with each tensor of shape :obj:`(2, batch_size, num_heads, sequence_length, embed_size_per_head)`): Contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``): Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of shape :obj:`(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``): Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: import torch from transformers import CTRLTokenizer, CTRLLMHeadModel tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = CTRLLMHeadModel.from_pretrained('ctrl') input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=input_ids) loss, logits = outputs[:2] )rr+rrr,rrGrrN.r)r`r contiguousrrr&)r>rrr+rrr,rlabelsrGZtransformer_outputsrZ lm_logitsrJZ shift_logitsZ shift_labelsZloss_fctZlossrrrrKs(7   zCTRLLMHeadModel.forward) NNNNNNNNT) rLrMrNr3rrrrrKrOrrr?rrs r)NN) riloggingZnumpyr$r Ztorch.nnr8rZconfiguration_ctrlrZ file_utilsrrZmodeling_utilsrr getLoggerrLloggerrjrrr/Moduler0rQrRr_ZCTRL_START_DOCSTRINGrrkrrrrrs<     3& 2I