D0j,h$ dZddlZddlZddlmZddlZddlmZddlm Z m Z m Z m Z m Z mZddlmZmZddlmZmZmZmZdd lmZmZdd lmZmZejeZed Z d e!de"fdZ#de!de$e!e!ffdZ%e &dgd d~dee!ej'd fdee"ej(dd fdee"ej(dd fdeddf dZ)de!de"fd Z*e &d!gd" ddee!dzej'd# fd$ee"ej(d%d&d' fd(ee"ej(d)d* fd+ee"ej(d,d-d. fd/ededdfd0Z+d1e!dzd/e!dzd$e"d(e"d+e"de!dzddfd2Z,de!d$e"d(e"d+e"de!dzddf d3Z-e &d4d5d6g ddee!ej'd7 fdeddfd8Z.e &d9gd: d~dee!ej'd7 fd;ee"ej(d fd?ee"ej(d@dA fdeddf dBZ/e &dCgdD ddee!ej'dE fd+ee"ej(d,d-dF fd;ee"ej(d fdGee"ej(dHdI fdJee0e!dzej(dK fdLee0e!dzej(dM fdeddfdNZ1e &dOgdP ddQee!ej'dR fdSee!ej'dT fdeddfdUZ2e &dVgdW ddXee!dzej'dY fdZee!dzej'd[ fd9ee"ej(d\ fd]ee"ej(d^d_ fd`ee"ej(dadb fdcee!dzej(dd fdeee!dzej(df fdGee"ej(dHdg fdJee0e!dzej(dh fdLee0e!dzej(di fdjee!dzej(dk fdlee"ej(dmdn fdoee"ej(dpdq fdree"ej(dsdtdu fdeddf dvZ3e &dwgdx ddyee!ej'dz fd{ee!dzej'd| fdeddfd}Z4dS)z7Contains commands to interact with buckets via the CLI.N) Annotated)logging) BUCKET_PREFIX BucketFile FilterMatcher_is_bucket_path_parse_bucket_path_split_bucket_id_and_prefix)SoftTemporaryDirectorydisable_progress_bars) SearchOptTokenOpt get_hf_api typer_factory) format_sizeprint_file_listing)OutputFormatWithAutooutz"Commands to interact with buckets.)helppathreturnc,|dS)Nzhf://) startswithrs c/home/rurouni/.hermes/hermes-agent/venv/lib/python3.11/site-packages/huggingface_hub/cli/buckets.py _is_hf_handler4s ??7 # ##argumentc|trt|S t|S#t$rt d|dtdwxYw)zParse a bucket argument accepting both 'namespace/name(/prefix)' and 'hf://buckets/namespace/name(/prefix)'. Returns: tuple: (bucket_id, prefix) where bucket_id is "namespace/bucket_name" and prefix may be empty string. zInvalid bucket argument: -. Must be in format namespace/bucket_name or znamespace/bucket_name)rrr r ValueError)rs r_parse_bucket_argumentr#8s =)),!(+++ *8444     8 8 8  8 8 8    s :&A create)zhf buckets create my-bucketz hf buckets create user/my-bucketz-hf buckets create hf://buckets/user/my-bucketz*hf buckets create user/my-bucket --privatez+hf buckets create user/my-bucket --exist-ok)nameexamplesF bucket_idzTBucket ID: bucket_name, namespace/bucket_name, or hf://buckets/namespace/bucket_nameprivatez --privatezCreate a private bucket.exist_okz --exist-okz3Do not raise an error if the bucket already exists.tokenct|}|trk t|\}}n3#t$r&}t jt|d}~wwxYw|r t jd|dtd|}|||r|nd|}tj d|j |j dS) zCreate a new bucket.r*Nz-Cannot specify a prefix for bucket creation: . Use namespace/bucket_name or namespace/bucket_name.)r(r)zBucket created)handleurl) rrrr#r"typer BadParameterstr create_bucketrresultr/r0) r'r(r)r*api parsed_idprefixe bucket_urls rr$r$IsD 5 ! ! !CM**  - 6y A A Ivv - - -$SVV,, , -  $W WW1>WWW  """,#J J (9z~NNNNNNs? A/ !A**A/c||tr|ttd}n|}d|vS)zFCheck if argument is a bucket ID (namespace/name) vs just a namespace.N/)rrlen)rrs r _is_bucket_idr>sA=))M**,,- $;rz list | ls) zhf buckets listzhf buckets list huggingfacez$hf buckets list --search "my-prefix"zhf buckets list user/my-bucketz!hf buckets list user/my-bucket -Rz!hf buckets list user/my-bucket -hz%hf buckets list user/my-bucket --treez(hf buckets list user/my-bucket --tree -hz+hf buckets list hf://buckets/user/my-bucketz%hf buckets list user/my-bucket/sub -RzyNamespace (user or org) to list buckets, or bucket ID (namespace/bucket_name(/prefix) or hf://buckets/...) to list files.human_readablez--human-readablez-hz$Show sizes in human readable format.as_treez--treez3List files in tree format (only for listing files). recursivez --recursivez-Rz0List files recursively (only for listing files).searchc|duot|}|r,|tjdt|||||dSt ||||||dS)zList buckets or files in a bucket. When called with no argument or a namespace, lists buckets. When called with a bucket ID (namespace/bucket_name), lists files in the bucket. Nz'Cannot use --search when listing files.)rr?r@rAr*) namespacerBr?r@rAr*)r>r1r2 _list_files _list_buckets)rr?r@rArBr* is_file_modes rlist_cmdrHsr4'CM(,C,CL  $%NOO O)        )       rrDc|rtjd|rtjd|K|tr1|t td}|d}t |}fd|||D}tj |ddd  dS) zList buckets in a namespace.z'Cannot use --tree when listing buckets.z,Cannot use --recursive when listing buckets.Nr<r,cg|]:}|j|jrt|jn|j|j|jd;S))idr(size total_files created_at)rKr(rrLrMrN).0bucketr?s r z!_list_buckets..se     )~@N_K ^<<T     Eu^WXabbbbbbsA B!BBinfozhf buckets info user/my-bucketz+hf buckets info hf://buckets/user/my-bucketzFBucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_namec t|} t|\}}n3#t$r&}tjt |d}~wwxYw||}tj|ddS)zGet info about a bucket.r,NrK)id_key) rr#r"r1r2r3 bucket_infordict)r'r*r6r7_r9rPs rr^r^s" 5 ! ! !C)-i88 11 ))) Q((()__Y ' 'FHVD!!!!!!s% A!AAdelete)z hf buckets delete user/my-bucketz-hf buckets delete hf://buckets/user/my-bucketz&hf buckets delete user/my-bucket --yesz-hf buckets delete user/my-bucket --missing-okyesz--yesz-yzSkip confirmation prompt. missing_okz --missing-okz3Do not raise an error if the bucket does not exist.c|trl t|\}}n3#t$r&}t jt |d}~wwxYw|r t jd|dtd|}n$d|vr t jd|dtdtjd|d | t| }| || tj d |dS)zDelete a bucket. This deletes the entire bucket and all its contents. Use `hf buckets rm` to remove individual files. Nz-Cannot specify a prefix for bucket deletion: r-r.r<zInvalid bucket ID: r!z(Are you sure you want to delete bucket ''?rer,)rfzBucket deleted)r') rrr#r"r1r2r3rconfirmr delete_bucketr5)r'rerfr*r7r8r9r6s rrdrd6sjJM**  - 6y A A Ivv - - -$SVV,, , -  $W WW1>WWW  I    a) a a;H a a a   KH9HHHcRRRR 5 ! ! !CiJ777J9555555s/ A!AAz remove | rm)z)hf buckets remove user/my-bucket/file.txtz2hf buckets rm hf://buckets/user/my-bucket/file.txtz.hf buckets rm user/my-bucket/logs/ --recursivez:hf buckets rm user/my-bucket --recursive --include "*.tmp"z8hf buckets rm user/my-bucket/data/ --recursive --dry-runzBucket path: namespace/bucket_name/path or hf://buckets/namespace/bucket_name/path. With --recursive, namespace/bucket_name is also accepted to target all files.z0Remove files recursively under the given prefix.dry_runz --dry-runz8Preview what would be deleted without actually deleting.includezQInclude only files matching pattern (can specify multiple). Requires --recursive.excludezLExclude files matching pattern (can specify multiple). Requires --recursive.c t|\}n3#t$r&}tjt |d}~wwxYw|dkr|stjddd|s|r|stjdt |} |r*t jd} g} | | d pdd D]R} t| tr;| | | d t| d S| d t| d |s|r t!||fd| D} n| } d| D}t#d| D}t%|d }|st jddSt|d|}|sP|sNt jdd|Dt jd|ddd|rMt jdfd|Dt jd|ddS| |t jd |dd!t||"dS| d }|stjd#|r7t jd$t0d |t jd%dSt jd&|d'd|| |gt jd(|)dS)*zbRemove files from a bucket. To delete an entire bucket, use `hf buckets delete` instead. Nz?No file path specified. To remove files, provide a path (e.g. 'zd/FILE') or use --recursive to remove all files. To delete the entire bucket, use `hf buckets delete z`.z,--include and --exclude require --recursive.r,zListing files from remoter<TrYzListing files from remote (z files))include_patternsexclude_patternscHg|]}|j|S)matchesr)rOfmatchers rrQzremove..s,MMM1W__QV5L5LMQMMMrcg|] }|j SrtrrOrvs rrQzremove..s444af444rc3$K|] }|jV dSN)rLrys r zremove..s$77A777777r)r?zNo files to remove.z file(s) totaling  c3 K|] }d|V dS)z Nrt)rOrs rr|zremove..s(BBt{D{{BBBBBBrzRemove z from 'rhFric36K|]}dtd|VdS)delete: r<N)r)rOrr's rr|zremove..s:ccPTK-KKKTKKccccccrz (dry run) z would be removed.)rdzRemoved ')r' files_deletedrLzFile path cannot be empty.rz"(dry run) 1 file would be removed.zRemove 'z' from 'z File removed)rr')r#r"r1r2r3rrstatusr]rT isinstancerappendupdater=donersumrtextjoinrjbatch_bucket_filesr5r)rrArerlrmrnr*r8r9r6r all_filesitem matched_files file_paths total_sizesize_str count_label file_pathr'rws @@rremoverss\z)28<< 66 ))) Q((()||I|  R  R RDM R R R   Q7QIQ !OPPP 5 ! ! !C<H788&( (( ==%%-)   U UD $ ++ U  &&& SC NNSSSTTT I#i..IIIJJJ  &g &#WwWWWGMMMM MMMMM%M44m444 7777777 z$???  H* + + + FZFFHFF  P7 P HTYYBBzBBBBB C C C KC+CCiCCC O O O O   HTYYccccXbccccc d d d HA+AAA B B B F y<<< 7{ 7 79 7 7 7j//       MM#&&  C$%ABB B   HF FyFF9FF G G G H9 : : : F ?y??)???SIIII y)=== > YGGGGGGs A!AAmove)z/hf buckets move user/old-bucket user/new-bucketz/hf buckets move user/my-bucket my-org/my-bucketzIhf buckets move hf://buckets/user/old-bucket hf://buckets/user/new-bucketfrom_idzMSource bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_nameto_idzRDestination bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_namecRt|\}}|r tjd|dtdt|\}}|r tjd|dtdt |}|||t jd||dS)z2Move (rename) a bucket to a new name or namespace.z)Cannot specify a prefix for bucket move: r-r.r,)rrz Bucket movedN)r#r1r2rr move_bucketrr5)rrr*parsed_from_id from_prefix parsed_to_id to_prefixr6s rrrs2#9"A"ANK   S S S-: S S S   5U;;L)   S S S-: S S S   5 ! ! !COON,O???J~~\JJJJJJrsync)z2hf buckets sync ./data hf://buckets/user/my-bucketz2hf buckets sync hf://buckets/user/my-bucket ./dataz;hf buckets sync ./data hf://buckets/user/my-bucket --deletez^hf buckets sync hf://buckets/user/my-bucket ./data --include "*.safetensors" --exclude "*.tmp"zIhf buckets sync ./data hf://buckets/user/my-bucket --plan sync-plan.jsonlz'hf buckets sync --apply sync-plan.jsonlzhf buckets cp my-config.json hf://buckets/user/my-bucket/logs/zKhf buckets cp my-config.json hf://buckets/user/my-bucket/remote-config.jsonz7hf buckets cp - hf://buckets/user/my-bucket/config.jsonzchf buckets cp hf://buckets/user/my-bucket/logs hf://buckets/user/archive-bucket/ # nests logs/ dirzihf buckets cp hf://buckets/user/my-bucket/logs/ hf://buckets/user/archive-bucket/ # copies contents onlyzehf buckets cp hf://datasets/user/my-dataset/processed/ hf://buckets/user/my-bucket/dataset/processed/srczMSource: local file, any hf:// handle (model, dataset, bucket), or - for stdindstz@Destination: local path, bucket hf://... handle, or - for stdoutc t|}t|}|duot|}t|}|duot|}|dk}|dk} |rf|rd |||n3#t$r&} t jt| d} ~ wwxYwtj d||dS|s.|s,|s*|t jdt jd|r|st jd|rC|rAt|\} } | d ks| d rt jd | r|st jd |s5|s3tj |rt jd |rGt|\} } | d ks| d rt jd| d dd}| rt!5t#5}tj ||}|| | |fgt)|d5}|dx}r;t,jj||dx};dddn #1swxYwYdddn #1swxYwYdddn #1swxYwYdSdS||}nvtj |s4| tjs| d r!tj ||}n|}tj |}|rtj|d|| | |fgtj d||dS|rht|\} }t,jj}|| ||fgtj dd|dStj |st jd|t|\} } | d kr tj |}n:| d r#| tj |z}n| }|| ||fgtj d|tB| d |dS)zCopy files to or from buckets.r,N-Copied)rrz2Missing destination. Provide a bucket path as DST.z;One of SRC or DST must be a bucket path (hf://buckets/...).z+Stdin upload requires a bucket destination.rpr<zAStdin upload requires a full destination path including filename.z"Cannot pipe to stdout for uploads.zNSource must be a file, not a directory. Use `hf buckets sync` for directories.zJSource path must include a file name, not just a bucket or directory path.r rbiHT)r) Downloaded)addUploadedstdinzSource file not found: )"rrr copy_filesr"r1r2r3rr5r endswithosrisdirrsplitr r rdownload_bucket_filesopenreadsysstdoutbufferwritesepdirnamemakedirsrrisfilebasenamer)rrr*r6 src_is_hf dst_is_hf src_is_bucket dst_is_bucket src_is_stdin dst_is_stdoutr9rcr8r'filenametmp_dirtmp_pathrvchunk local_path parent_dir remote_pathdatas rrrsK4 5 ! ! !Cc""I46M#$6$6I#C((MtO<(<('(( ; ;+--;!w||GX>>H--i68:L9MNNNh--;'(vvj'9'99e;J-33E:::()vvj'9'99e;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;{% s## !s||BF';'; !s||C?P?P !W\\#x88  44J 7 J6666  % %i6:2F1G H H H J|* = = = = = = Y!3C!8!8 ;y$$&& yk/B.CDDD :7444444w~~c"" F$%Ds%D%DEE E.s33 6 R<<'**3//KK __S ! ! ! 27#3#3C#8#88KK K y[/A.BCCC :3},Wi,W,W+,W,WXXXXXXst&A== B-!B((B-K<%A K%/AK K%K K%K K% K<%K) )K<,K) -K<<LL)FFN)NFFFNNr{)FFFNNN)NNFFFNNFNNNFFFN)NN)5__doc__rrtypingrr1huggingface_hubrhuggingface_hub._bucketsrrrrr r huggingface_hub.utilsr r _cli_utilsrrrr _file_listingrr_outputrr get_logger__name__logger buckets_clir3boolrtupler#commandArgumentOptionr$r>rHrFrEr^rdr\rrrrrtrrrs >=  ######  ;:::::::........  H % %m!EFFF $$$$$$ S U38_    "    .  ++O+O g    +O  +    +O  F    +O* ++O, -+O+O+O  +O\CD     0    E> >  d W    >    7    > $  F    %> 2   C    3> B C> D E> F G> > > > B KTz K $J K K K  K : K  K K K KFcccc c : c  cccc:  (5"" Y    " " """"*  .  -1616 Y    16    ,     16  F    16, -16. /16161616h    6    [@H@H a    @H   C    @H$    ,     %@H4  K    5@HB S D d    C@HN S D _    O@HZ [@H\ ]@H@H@H  @HF  *"K"K  `    "K  e     "K "K "K"K"K"KT       &         yrPrP  d ^     rP  d c     rP   B     rP&  N    'rP4  O    5rPB  d  P     CrPN  d  6     OrPZ  H    [rPh S D I    irPt S D I    urP@ d  ;    ArPL  T    MrPZ  V    [rPh   8    irPx yrPz {rPrPrP  rPt     , lYlY  ^U^!p q q qq lY  d NEN(jkkkk  lY lY lYlYlY! lYlYlYr