U
    Jc"                     @   s   d dl mZ d dlZd dlm  m  m  mZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlm  mZ d dlmZmZmZ dd	lmZmZ dd
lmZ erd dlmZ eG dd deZ dS )    )	dataclassN)ShardMetadata)Shard)!_parse_and_validate_remote_device)narrow_tensor)ListUnionTYPE_CHECKING   )get_chunked_dim_sizeget_split_size)ShardingSpecShardedTensorc                   @   s   e Zd ZU dZeeef Zeed< e	ee
jjef  ed< dd Zedd Ze
jejejdd	d
Zde
jeddddZdS )ChunkShardingSpeca  
    This is a type of PlacementSpec that defines the placement as being sharded
    across multiple devices. In particular, it represents sharding a Tensor
    along a single dimension into equal chunks (similar to :meth:`torch.chunk`).

    The semantics of how a tensor is partitioned is inline with
    :meth:`torch.chunk`, where ``dim`` in torch.chunk corresponds to the
    specified ``dim`` and ``chunks`` in torch.chunk is the number of elements
    in the placement specified.

    Args:
        dim (int or str):
            The dimension to shard on, could be an integer representing the
            dimension or a string in case of named tensors where dimensions are
            named. Note that named tensor support is not added yet.
        placement(List[Union[_remote_device, str]]):
            Specifies the placement of each shard of the Tensor. The size of
            the list represents the number of shards to be created. This could
            be a list of
            :class:`torch.distributed._remote_device`'s. This list
            could also contain a string which represents remote
            device as accepted by
            :class:`torch.distributed._remote_device`
    dim
placementsc                 C   sD   |  | j t| jD ](\}}t|tjjstj|| j|< qd S )N)_verify_dimr   	enumerater   
isinstancetorchdistributed_remote_device)selfiZremote_device r   ^/tmp/pip-unpacked-wheel-gikjz4vx/torch/distributed/_shard/sharding_spec/chunk_sharding_spec.py__post_init__9   s    zChunkShardingSpec.__post_init__c                 C   s.   t | trtdt | ts*td|  d S )Nz7ChunkShardingSpec does not support named dimension yet!z,Sharding dim needs to be an integer, found: )r   strNotImplementedErrorint
ValueError)r   r   r   r   r   ?   s    

zChunkShardingSpec._verify_dim)tensor_sizestensor_propertiesreturnc                 C   s   t |}| | j | j|ks*| j| k r:td| j g }|| j }t | j}t||}t| jD ]^\}}	t|||}
|
dkrft|}dg| }|| || j< |
|| j< t	|||	d}|
| qft|||S )NzInvalid sharding dim: r   )Zshard_offsetsshard_sizes	placement)lenr   r   r!   r   r   r   r   listr   appendsharded_tensor_metaShardedTensorMetadata)r   r"   r#   Ztensor_num_dimshards_metadatasharding_dim_sizechunks
split_sizeidxr&   Zchunked_dim_sizeZ
shard_sizeZcurrent_offsetsZshard_metadatar   r   r   build_metadataM   s4    




z ChunkShardingSpec.build_metadatar   Nr   )tensorsrc_rankr$   c                 C   s  ddl m} tj|j|j|jtj|	 d}t
|}| | |}g }d}	d}
dgt
| }| | j }t| j}t||}t| }||| j< |jD ]}t||j\}}||krt||}|j| j |k r|  |}n|   }|||< ||krtj||j|j|d}	|}
q|	dk	s6t|
dk	sDt|}|dk	rl|t  k	rlt!||}t
j"|	||kr|nd||d t|	 |
jkr|	|
j }	|j|	_|#t$|	|
d |j%|||d}| |_&|S )	z
        Args:
            src_rank: group rank relative to ``process_group``

            N.B. If ``process_group`` is None, ``src_rank`` is a global rank.
        r   r   )dtypelayoutrequires_gradZmemory_formatZ
pin_memoryN)r4   r5   device)Zscatter_listsrcgroup)r2   metadata)process_group)''torch.distributed._shard.sharded_tensorr   r*   TensorPropertiesr4   r5   r6   r   Zcontiguous_format	is_pinneddistZget_rankr1   sizeZget_world_sizer   r'   r   r   r(   r,   r   r&   r   r%   detachcloneZresize_
contiguousemptyAssertionErrordistributed_c10dZ_get_default_groupZget_global_rankZscatterr)   r   Z+_init_from_local_shards_and_global_metadataZ_sharding_spec)r   r2   r3   r;   r   r#   Zcurrent_rankZtensor_metaZlocal_shardsZlocal_tensorZlocal_metadataZtensors_to_scatterr-   r.   r/   Zscatter_shapeZ
shard_metaZrankr7   Znarrowed_tensorZtensor_to_scatterZsrc_for_scatterstr   r   r   shardt   sr    






   zChunkShardingSpec.shard)r   N)__name__
__module____qualname____doc__r   r    r   ZShardingDim__annotations__r   r   r   r   r   staticmethodr   Sizer*   r=   r+   r1   ZTensorrH   r   r   r   r   r      s   

'r   )!Zdataclassesr   r   Z0torch.distributed._shard.sharded_tensor.metadatar   Z_shardZsharded_tensorr:   r*   Z!torch.distributed._shard.metadatar   Z-torch.distributed._shard.sharded_tensor.shardr   Z-torch.distributed._shard.sharded_tensor.utilsr   Ztorch.distributed._shard._utilsr   Ztorch.distributedr?   Z"torch.distributed.distributed_c10drF   typingr   r   r	   Z
_internalsr   r   apir   r<   r   r   r   r   r   r   <module>   s   