+jo ddlZddlZddlZddlZddlZddlmZddlmZm Z m Z ddl m Z ddl Z ddl mZddlZddlZddlmZmZddlmZddlmZddlmZmZd d lmZmZm Z m!Z!ej"d d Z#ej$e# %dZ&dZ'dZ(dZ)dZ*GddZ+dZ, d#dZ-dZ.GddZ/ddgddfde0e1e2e1fde1de2e3d e4d!e4f d"Z5dS)$N)stderr)OptionalDictUnion)tqdm)infer_auto_device_mapdispatch_model)get_balanced_memory)snapshot_download) load_file save_file) get_codecget_32bit_codecget_lutsencode_weightsdfloat11z decode.ptx)pathdecodez0.5.0)i)encoded_exponent sign_mantissac,eZdZdZiZedZdS) TensorManagerz Static utility class that manages tensor allocation and reuse to minimize memory allocation overhead during tensor reconstruction. ct|trtj|}|tjvr_tj|}||kr |d|Stj|=tjtj |tj |}td|d|t|tj|<|S)ao Get a bfloat16 tensor with at least n_elements on the specified device. If a tensor already exists on the device and is larger than n_elements, a slice of the tensor with exactly n_elements is returned. If n_elements exceeds the size of the existing tensor, the existing tensor is deallocated and a larger one is allocated. Args: device: The device to allocate the tensor on (e.g., 'cuda:0') n_elements: The exact number of elements required Returns: A bfloat16 tensor with exactly n_elements on the specified device Ndtypedevicez Allocated z bf16 on device file) isinstancestrtorchrr_tensorsnumelcuda empty_cacheemptybfloat16printr)r n_elementsexisting_tensor new_tensors T/home/wildlama/comfy/ComfyUI/.venv/lib/python3.11/site-packages/dfloat11/dfloat11.pyallocate_bfloat16zTensorManager.allocate_bfloat16<s$ fc " " *\&))F ]+ + ++4V&QQQ  ?:??v??fMMMM*4 v&N)__name__ __module__ __qualname____doc__r$ staticmethodr/r0r.rr4s> H''\'''r0rc4tfd}|S)a Creates a PyTorch forward pre-hook that decodes compressed DFloat11 weights on-the-fly. This hook reconstructs full-precision weights from compressed representations using a custom CUDA kernel during the forward pass. Args: threads_per_block: CUDA thread configuration bytes_per_thread: Number of bytes processed per CUDA thread Returns: A forward pre-hook function for PyTorch modules cv|jj}t|drs|jD]Y\}}t||rt ||j|ks+||||dZ|j }|j }|jj d}t ||}ttj|dzz f} t"j|j5t+| |j|j|j |j|j|j||||g dddn #1swxYwYt5|t6jr&||j|j|_ nt5|t6j!r&||j"|j#|_ nZtIj%||j&} tO|j(| D]*\} } | | j| j| _ +t|drM|j)D]5}t||r!t ||} tU||~ 4dSdS)Noffloaded_tensorsT) non_blockingrgridblock shared_memargs)+lutsrhasattrr9itemsgetattrregister_buffertorr%rshaperr/intmathceilcpr&Deviceindex_decodeshared_mem_sizedata_ptroutput_positionsgapsr!nnLinearview out_features in_featuresweight Embeddingnum_embeddings embedding_dimr# tensor_splitsplit_positionszipweight_injection_moduleskeysdelattr)module_r tensor_nametensorr+n_bytesn_luts reconstructedblocks_per_gridweights sub_modulerWtmpbytes_per_threadthreads_per_blocks r. decode_hookzget_hook..decode_hookwsf# 6. / / ^'-'?'E'E'G'G ^ ^# VFK00^6=fk6R6R6Y]c6c6c**; &W[ 8\8\]]])//11 )//11"1%&77 KK ty4Ea4HK[4[)\]]^^aW^^FL ) )   0AfNd $$&&'0022$--//'0022 $$&&&&((l                    fbi ( ( a)..#V%7FMM - - a)..%v';FMM (8NOOG&)&*I7&S&S a a" F$*KK 0GI_$`$` !! 6. / / %7<<>>   6;//!&+66CFK000     sB'G55G9<G9)tuple)rmrlrns`` r.get_hookrpgs</00777777r r0FTc T |d}|d}|d} |sdtj|Dn|g} d} |r| dz } |r| dz } | dz } t| | D]} |s tj|| n| } t | }|D]x\}}||vr|t| vr|t| |}|j |j kr|j |td |d |j d |j t t||}|j |j kr| | td |d |j d |j t 5|d}|}t#|ddD]A\}}t%||rt'||}&td|t n|ddkr$t)|d|n|r||dkr|dt,vrt%|dst)|di|r|n||j|d<|5|dkr/t3|jt3t,kr|dz}n||d||ddkrK|t9||| D]\}}t;j|d|ddrt?|t@j!r|j"}tG|d~it?|t@j$r|j"}tG|d~t)|dg|D]`}|d}|}|D]}t'||}|j"}tG|d~|j%&|a|ddkr|'tPj)*}t)|d|ddzdz|dd|ddz +,dzzz|S)a Loads DFloat11 compressed weights from safetensors files and configures the model to use them with on-the-fly decompression. Args: model: The PyTorch model to load weights into directory_path: Path to the directory containing safetensors files dfloat11_config: Configuration for DFloat11 compression Returns: The model with configured DFloat11 compression rmrl pattern_dictc<g|]}|d|S) .safetensors)endswith).0fs r. z,load_and_replace_tensors..s9N1K1K r0zLoading DFloat11 safetensorsz (offloaded to CPUz, memory pinned))desczShape mismatch for z: model z vs loaded r.NzCannot find module path for r\rr9rrrWr^rPrN)-oslistdirrrjoinr rB state_dictdictnamed_parametersrFdatacopy_r*r named_bufferssplit enumeraterArCsetattrtolistoffloaded_tensor_names pin_memoryr9lenrDregister_forward_pre_hookrpre fullmatchr!rRrXrWr`rSr^appendrTr#uint32numpymaxitem)modeldirectory_pathdfloat11_config cpu_offloadcpu_offload_blocksrfrom_single_filermrlrrsafetensors_files loading_desc file_name file_pathloaded_tensorsrc tensor_valueparambufferpartsraipartpattern attr_namesrk attr_pathtargetpoutput_positions_nps r.load_and_replace_tensorsrsJ*((;<'(:;'7L 4:n--$2#32L,,  . - -L +,???[[ CSbBGLL;;;Yb #9--*8)=)=)?)?T T  %Ke..0000$u'='='?'?"@"@@@ !7!7!9!9::;GE{l&888 ((6666uKuuuuamasuu}CDDDDD"%"5"5"7"788EF||'999 \2222vKvvvvbnbtvv~DEEEEE$))#.. )ss44<<GAtvt,,!(!6!6J[JJQWXXXXRy$555(9<;N;N;P;PQQQQ& L,>,FJ\_`J`J`fklnfotJgJgJ#*63F#G#GI '0CR H H H_iB{,BYBYB[B[B[o{F4U2Y? 2 >EWZ[E[E[beflf~bbDGH^D_D_c_c_ 2a 7 2#2259lKKKRy$66688BSUe9f9fggg4@3E3E3G3GWW/GZ!|GSXXeCRCj5I5IJJW#-fbl#C#C!W*0-C$+FH$=$=$=(+%/ %B%B!W*0-C$+FH$=$=$=(+%,F4NPR$S$S$S5? %W %W 090D0D1716)H)HA5r d| vr | d}n'tA| dr| j!}ntEdtG|| ||||| |sSd}|$%D] }||j&z } tOd|dz ddtP|r|)|}n|dks JdtU||d}tW|||}tY|||}t[||}t]d|/DrtOdtP|S) aU Load a model with DFloat11 compressed weights from local path or Hugging Face Hub. Args: dfloat11_model_name_or_path: Local path or HF Hub model name device: Target device for the model device_map: Strategy for distributing model across devices max_memory: Maximum memory allocation per device bfloat16_model: Optional pre-initialized model to load weights into cpu_offload: Enables CPU offloading; only keeps a single block of weights in GPU at once cpu_offload_blocks: Number of transformer blocks to offload to CPU; if None, offload all blocks pin_memory: Enables memory-pinning/page-locking when using CPU offloading from_single_file: Whether to load a single safetensors file pattern_dict: Dictionary mapping regex patterns to submodule lists **kwargs: Additional arguments passed to AutoModelForCausalLM.from_config Returns: Model with DFloat11 compressed weights configured for on-the-fly decompression zeExpected `dfloat11_model_name_or_path` to be the path to a safetensors file, but found a directory: "".z The file "z" does not exist./__) local_dirrversionrmrlrr config.jsonrutf-8encodingNr)AutoModelForCausalLM AutoConfigGenerationConfig)no_init_weights torch_dtypezd"dfloat11_config" not found: it is expected to be found in the config file or passed as an argument.)rrrrzTotal model size: geAz0.4fz GBrrz>device_map should be 'auto' if no specific device is provided.rr)rno_split_module_classesc36K|]}|jjdkVdS)cpuN)rtype)rvrs r. z0DFloat11Model.from_pretrained..s,NN%5<$-NNNNNNr0zqWarning: Some model layers are on CPU. For inference, ensure the model is fully loaded onto CUDA-compatible GPUs.)0rrisfileisdirIsADirectoryErrorFileNotFoundErrorexistsreplacer rrmrlopenrjsonload transformersrrrtransformers.modeling_utilsrfrom_pretrained from_configr#r) tie_weightsevalgeneration_config Exceptionr!rrArAttributeErrorrrvaluesnbytesr*rrErr rr any parameters)clsrrrrbfloat16_modelrrrrrrkwargsdfloat11_model_pathconfigrwrrrrrrer model_bytesrrs r.rzDFloat11Model.from_pretrainedVsF  bw~~9:: e&A##:;; e')pQl)p)p)pqqq'(c5P(c(c(cdddw~~9:: b&A##&A&I&I#t&T&T#w~~&9::b%&AM`aaaa !  *%#*->,<(4 (("',,':MJJCZabbb*fg!Yq\\F***************#EE W W W W W W W W W W C C C C C C //0CDDF ""  8,8(-:@!!###                 $4$D$DEX$Y$Y!*;''     fd # # I(9V(C(C$%67OO V. / / I$4OO "HII I ! &#8J!4D     QK))++2244 , ,u|+ B{S'8BBBB P P P P  XHHV$$EE''')i'''3E?>;Z[[ ,UzcstttJ.ueuvvvJ"5*55ENN5;K;K;M;MNNNNN XJQWXXXX s7EEE=AG  GGG55 H H rc @||||||||| d| S)NT) rrrrrrrrrrr)r) rrrrrrrrrrrs r.rzDFloat11Model.from_single_files?""(;!!)#1!!%#   r0) NrNNFNTFN)NrNFNT)r1r2r3r4 classmethodr"rrrrGboolrlistrrr6r0r.rrPs !% GK!,0!&7;{{%({ { { T%S/5c?"BCD {{%SM{{{tCcN34{{{[{| !% GK!,0    3S >*     T%S/5c?"BCD  %SM    [   r0ri'rr save_path block_rangesave_single_filecheck_correctnessc12tj|dd}d}|D]F\}} |D]*\2} t j|2r|sUd2vr@2dd\} } || } t| | dnt|2d|dz }||dkr||dkrd}ng}t| tj r| j j jtjksJd2d| j j j|| j j t+| d nt| tjr| j j jtjksJd2d| j j j|| j j t+| d n| D]}|d}| }|D]}t1||}|j j jtjks"Jd2d|d|j j j||j j t+|d t3tj|\}}t7|\}}}|t;|}t=||t>t@d\}}}}}|rtj!d 1|j"d}|#}|#} tItK1fd |||||g\}!}"}#}$}%tj&|tj1 }&tOtQj)| t@dt>zz f}'|*tj+,}(t@dd zd z|(dd|(ddz -.dzz})t_d|)dt`j121j35ti|'t@|)|!5|"5|#5|$5|%5|&5|| |g dddn #1swxYwYtj||&k6.}*|*rt_dntod2d| 8d|| 8d|| 8d|| 8d|*tj9| 8d|| 8d||sy| :}+2fd|+D}+tw|+tj<=|2>dddz,H|rt~t@t>|d},t|d r |,|jA_Bn#t$rYnwxYwt|d!r|s|D|n@tw|:tj<=|d"d}-tj<Etj<=|d#r}ttj<=|d#d$d%&5}.tjH|.}/d'|/vrt|/d'trd}-dddn #1swxYwY|-rdttj<=|d#d(5}0tjJd'|,i|0d)ddddS#1swxYwYdSdSdS)*NT)exist_okrr{rFz@Expected weights to be in bfloat16 format for compression, but 'z ' has dtype rWzcuda:0c.|S)N)rE)xrs r.z compress_model..7s#uvuyuy{AvBvBr0rr}r|r~zUsing z bytes of shared memory.r;uN✅ Correctness check passed: decompressed weights match the original weights.ue❌ Correctness check failed: The decompressed weights do not match the original weights for module "rr@rrrPrQr\c&i|] \}}d||S)r{r6)rvkeyvaluers r. z"compress_model..Zs-!c!c!c*#uY"6"6"6"6!c!c!cr0rbrtrrsave_pretrainedzmodel.safetensorsrrrrrw)indent)KrmakedirsrBrrrrsplit get_submodulerr!rRrXrWrrr#r)rdetachrflattenr`rSrrCrcatrprint_code_tablerrrlrmrrFr%rmapr(rGnprIrTrrrrr*rJr&rKrLrMrOall RuntimeErrorrDuint8rr rrrrrArrrrrrrrrdump)3rrrrrrr block_index save_modelrrrj parent_name child_nameparentrirrrr_codec_countercodecrbtabler@encoded other_8bitsrPrQr\rfr+re cuda_luts cuda_encodedcuda_other_8bitscuda_output_positions cuda_gaps cuda_outputsrhrrN _is_correctrr save_configrwr config_filerrs3 @@r.compress_modelr$s K D))))KJ+1133_q_q%*%8%8%:%:] q] q !Iz|GY//\ q'8i''2;2B2B32J2J/ Z!&!4!4[!A!A D9999y$777q +a.00 ;q>11!&JEj",772%,175>IIIQ[dQQr|sDsIsOQQJIINN:#4#9#@#@#B#B#F#F#H#H#P#P#R#RSSSJ1111 BI662%,175>IIIQ[dQQr|sDsIsOQQJIINN:#4#9#@#@#B#B#F#F#H#H#P#P#R#RSSSJ1111&0 2 2 ) 4 4!+!&88A%,VQ%7%7FF%}175>III]_h]]kt]]CICPCUC[]] JIIv}'9'@'@'B'B'F'F'H'H'P'P'R'RSSS1111#,UYw-?-?#@#@ "1(";";q%&&(((P^_fhmoBSTUBVQWQWM&6o$b"\(33F!Z]F!,!2!2!4!4J%mmooGbfgjlBlBlBlBEIKRT_aqswDxhyhyczcz_I|-=?TV_#(;zX^#_#_#_L'*277>OPQ>RUe>e3f+g+g'h'h&kO*:*?*? *M*M*S*S*U*U'&7&:Q&>&BFYZ[Z\Z\F]`stwuwtw`xFxE}E}EEFEFEFGFGJKFK'KOL?LLLMMM 55  _ ,,     5+ , , X5E X  ! !) , , , , e&&(("',,yBU*V*V W W W 7>>"',,y-@@ A A (bgll9m<WY]3^3^."'K  ( ( ( ( ( ( ( ( ( ( ( ( ( ( (  *bgll9m<r3s (((((((((( <<<<<<<<000000------22222222PPPPPPPPPPPP += *:| D D ",H % % % 2 28 < <  00000000fIII`AAAAH2] ] ] ] ] ] ] ] H Z!" L*L*sDI~&L*L*c L*  L*  L*L*L*L*L*L*r0