U
    Jcu                     @   s  d dl mZmZ d dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ dd	lmZmZmZmZmZmZ dd
lmZmZmZmZmZmZ ddlmZ ejedddZej edddZ!eedddZ"eeedddZ#e$eeedddZ%e$ej edddZ&e$eddd Z'd!d" Z(d#d$ Z)e$eee ee d%d&d'Z*eed(d)d*Z+e$eee d+d,d-Z,e$eeee d.d/d0Z-dS )1    )ListAnyN)ShardMetadata)ShardedTensor)TensorProperties)Shard)"_check_shard_metadata_pair_overlap   )LoadItemTypeSavePlanReadItem	WriteItemWriteItemTypeTensorWriteData)BytesStorageMetadataChunkStorageMetadataTensorStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPES)+_shards_get_overlap_region_wrt_saved_tensor)sizereturnc                 C   s   t dgt|  t| dS )Nr   )shard_offsetsshard_sizes)r   lenlist)r    r   W/tmp/pip-unpacked-wheel-gikjz4vx/torch/distributed/_shard/checkpoint/planner_helpers.py_create_shard_metadata$   s    r   )tensorr   c                 C   s   t | t|  dS )N)r    metadata)r   r   r   )r    r   r   r   _create_shard_from_tensor*   s    
r"   )shard_mdr   c                 C   s   t t| jt| jdS )Noffsetssizes)r   torchSizer   r   )r#   r   r   r   _chunk_for_shard0   s    

r)   )sharded_tensorr#   r   c                 C   s   t t||  j|  jdS )NchunkZ
propertiesr   )r   r)   r!   Ztensor_propertiesr   )r*   r#   r   r   r   _sharded_tensor_metadata6   s
    r-   )fqnr*   r#   r   c                 C   s(   t |j}tt| |tjt||dS )NindextypeZtensor_data)r'   r(   r   r   r   r   ZSHARDr-   )r.   r*   r#   r%   r   r   r   _create_write_item_for_shard=   s    r2   )r.   r    r   c                 C   sN   t dgt|  }tt| |tjtt	|| dt
|| ddS )Nr   r$   r+   r/   )r'   r(   r   r   r   r   r   TENSORr   r   r   Zcreate_from_tensor)r.   r    r%   r   r   r   _create_write_item_for_tensorE   s    r4   r.   bytesc                 C   s   t t| tjdS )N)r0   r1   )r   r   r   BYTE_IOr5   r   r   r   _create_write_item_for_bytesioT   s    r8   c              	   C   s.   t tj| t|f|t|ft|fdS N)r1   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r
   r7   r'   r(   r:   Zdest_offsetr<   Zstorage_offsetlengthr   r   r   _create_read_item_for_byteioZ   s    


rA   c              	   C   s(   t tj| t||t|t|dS r9   )r   r
   r3   r'   r(   r:   r;   r<   r=   r>   r   r   r   _create_read_item_for_tensord   s    rC   )r.   checkpoint_mdlocal_shardsr   c                 C   s   g }t |D ]\}}t |jD ]\}}tt|jt|jd}t|j|sLqg }	g }
g }t||jdD ]*\}}}}|		| |
	| |	| qf|	t
t| |jj||
t| |j||	|d qq|S )N)r   r   )Zsaved_shardZcurrent_shardrB   )	enumeratechunksr   r   r&   r%   r   r!   r   appendrC   r   r   )r.   rD   rE   Z
read_itemsidxshardZstorage_idxZ
storage_mdZshard_md_from_storager=   r;   r>   ZdimZoffset_for_saved_tensorZoffset_for_current_tensorr@   r   r   r   _create_sharded_read_itemsn   sJ      

	rK   )
state_dictr   c                 C   sz   g }|   D ]d\}}t|trB| jD ]}|t||| q(qt|tjr`|t	|| q|t
|| qt|S )N)items
isinstancer   r!   Zshards_metadatarH   r2   r'   Tensorr4   r8   r   )rL   requestsr.   objr#   r   r   r   "_create_default_metadata_only_plan   s    
rR   )r.   objectr   c                    sJ   t tr" fdd D S t tjr:t gS t gS d S )Nc                    s   g | ]}t  |jqS r   )r2   r!   ).0rJ   r.   rS   r   r   
<listcomp>   s     z'_create_write_items.<locals>.<listcomp>)rN   r   rE   r'   rO   r4   r8   rU   r   rU   r   _create_write_items   s
    
rW   )r.   mdrQ   r   c                 C   s|   t |tr&tt| dt| dddgS t |tr:| }n6t |tjrRt|g}nt	d|  ddt
|  t| ||S )Nr   r?   z Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found )rN   r   rA   r   r   rE   r'   rO   r"   
ValueErrorr1   rK   )r.   rX   rQ   rE   r   r   r   _create_read_items   s,    



rZ   ).typingr   r   r'   Z!torch.distributed._shard.metadatar   Z'torch.distributed._shard.sharded_tensorr   Z0torch.distributed._shard.sharded_tensor.metadatar   Z-torch.distributed._shard.sharded_tensor.shardr   Z1torch.distributed._shard.sharding_spec._internalsr   Zplannerr
   r   r   r   r   r   r!   r   r   r   r   r   r   Z
reshardingr   r(   r   rO   r"   r)   r-   strr2   r4   r8   rA   rC   rK   rR   rW   rZ   r   r   r   r   <module>   s4    	 	
.