
    
9jn                     V    S SK r S SKrS SKJr  S SKJr  S SKJr  S rS rS r	S r
S	 rg)
    N)HuffmanCodec)tqdm)copyc                    0 nU R                  [        R                  5      nUS-	  S-  n[        R                  " USS9u  pE[	        UR                  5       UR                  5       5       H	  u  pgXqU'   M     [        R                  " U5      nX4$ )N      T)return_counts)viewtorchint16uniqueziptolistr   from_frequencies)	weightcounterWexponent_8bitsvalsfreqsvfcodecs	            Q/home/wildlama/miniconda3/lib/python3.13/site-packages/dfloat11/dfloat11_utils.py	get_codecr      sx    GEKK A1f_N,,~TBKDDKKM5<<>2
 3 ))'2E>    c                    [         R                  " U 5      nUR                  5       nSnUR                  5        H  u  nu  pT[	        X55      nM     UnU nSn[
        R                  " [        U R                  5       5      5      n	US:  a  [
        R                  " X5      S U n
US-  n[
        R                  " [        U R                  5       5      5      U
   n[        U 5      nU H  nSX|'   M	     [         R                  " U5      nUR                  5       nSnUR                  5        H  u  nu  pT[	        X55      nM     [        US-
  U5        US:  a  M  XgU4$ )Nr             )r   r   get_code_tableitemsmaxnparraylistvaluesargpartitionkeysr   print)r   r   tablemax_len_lcompressed_codeccompressed_countermin_kfreqmin_indicesmin_keysks                r   get_32bit_codecr6   %   s;   ))'2E  "EG[[]	6Ag/ #  E88D)*+D
B,ood26E:
88D01+>!']A$%! '889KL //1IAv'oG ' 	eai! B,  66r   c           	         S/nU R                  5        Ha  u  nu  p4[        U[        5      (       d  M  [        U5      SS  R	                  US5      S US-
  S-  S-   nXQ;  d  MP  UR                  U5        Mc     UR                  [        S9  [        R                  " [        U5      S4[        R                  S9n[        U5       GH1  u  px0 n	[        U5      S-  n
U R                  5        H  u  nu  p4[        U[        5      (       d  M  [        U5      SS  R	                  US5      nUR                  U5      (       d  MU  US-
  S-  U
:X  a$  [        XS-  S  R                  SS5      S5      nUnO4[        XS-  U
S-  S-    S5      nSUR                  US U
S-  S-    5      -
  nX;   a  X   U:w  a  [        S	U S
U	 35      eXU'   M     [!        U	5        [#        S5       H  nX;   a  X   nWXgU4'   M     GM4     [        R                  " S[        R                  S9nU R                  5        H&  u  nu  p4[        U[        5      (       d  M  UUSU4'   M(     [$        R&                  " [        R(                  " UU4SS95      $ )N r   0r       )key   dtypezKey z already exists in )r    r<   r   )axis)r"   
isinstanceintbinrjustappendsortlenr$   zerosuint8	enumerate
startswithljustindex
ValueErrorr*   ranger   
from_numpyconcatenate)r+   prefixesr;   bitsvalprefixlutspip
bytes_dictplbin_valdict_key
dict_valueicurr_vallenss                    r   get_lutsra   D   sU   tH!KKM[dc3Xab\''c23IdQh1_q5HJF%'	 * MMcM88S]C(9D8$
Vq[ %C$#s##c(12,,,T37%%a((qQ",#&wQy'9'?'?3'G#K%(
#&wQa!'Eq#I%(8>>'-BFQJ:P+Q%Q
-*2F*2T(4z9LZL)YZZ/98, !." 	jsA%="DQK - %6 88HBHH-D!KKM[dc3 DSM * 
d|!, r   c                 d   / n/ n/ nSnSnSn	Sn
[        U 5       H  nU	SU-  -  S-   [        U5      :  a  UR                  XSU-  -  SU-  -  -
  5        U	SU-  U-  -  S-   [        U5      :  a  UR                  U
5        UR                  U   u  pX|-  U-   nX-  nX-  n	U
S-  n
US:  d  M  XxS-
  -	  nUR                  U5        X~US-
  -  -
  nUS-  nUS:  a  M/  M     US:  a  U	SU-  -  S-   [        U5      :  a  UR                  XSU-  -  SU-  -  -
  5        U	SU-  U-  -  S-   [        U5      :  a  UR                  U
5        UR                  UR                     u  pX|-  U-   nX-  nUS:  a  XxS-
  -	  nOUSU-
  -  nUR                  U5        UR                  [        U 5      5        [        [        R                  " [        U5      X2-  -  5      5      nUR                  S/X?-  [        U5      -
  -  5        U Vs/ s H  n[        US5      PM     nnU VVs/ s H  nU  H  n[        U5      PM     M     nnn[        R                  " [        U5      [        R                  S9[        R                  " U5      [        R                  " U[        R                  S94$ s  snf s  snnf )Nr   r:   r    05br=   )r   rG   rE   _table_eofrB   r$   ceilextendformat
frombufferbytesrI   packbitsr%   uint32)datar   bytes_per_threadthreads_per_blockencodedgapsoutput_positionsbuffersize
total_sizeelement_countsbr   byteblocks_per_gridgapbinary_str_gapsbinarybitbinary_gapss                        r   encoder   v   s   GDFDJM$Z!../!3c$i?KK
A8H4H%IQQaMa%bbc!..1BBCaG#N^J__##M2||A+"	
aiQh'DNN4 q12FAID	 ai $ ax!../!3c$i?KK
A8H4H%IQQaMa%bbc!..1BBCaG#N^J__##M2||EJJ'+"	19Qh'Da$h'DtCI&"''#g,2C2V"WXYOKK(:SYFGH59:Tcvc5)TO:*9L/Vc3s8V8/KL==wrxx8"++k:RTVT\T\]muwu~u~T ;Ls   J'!J,c                    [         R                  " [         R                  " [        [	        S U 5      5      5      SS9S S n[         R
                  " U 5      R                  [         R                  5      nUS-	  S-  R                  [         R                  5      nUS-	  S-  US	-  -  R                  [         R                  5      n[        [	        [         R                  [        UR                  5       XU5      5      5      u  pn
[        S
UR                  5       UR                  5       -   U
R                  5       S-  -   U	R                  5       -   S-  UR                  5       S-  -  S S35        XXU4$ )Nc                 "    U R                  5       $ )N)numel)xs    r   <lambda> encode_weights.<locals>.<lambda>   s
    qwwyr   r   )dimr?   r   r   r:         zCompression factor:    d   r   z.2f%)r   cumsum
LongTensorr&   mapcatr
   r   torI   rP   r   r   r*   r   )weightsr   rn   ro   split_positions
W_combinedr   other_8bitsrp   rq   rr   s              r   encode_weightsr      sr   ll5#3#3D=PRY9Z4[#\bcdehfhiO7#((5J!Q$.225;;?N!Q$.*t2CDHHUK&*3~$$&ARS, '#G#
 
 '--/K4E4E4G"GJZJ`J`JbefJf"fimisisiu"uy|!|  AK  AQ  AQ  AS  VW  AW  "X  Y\  !]  ]^  _  `!1HHr   )r   numpyr$   	dahuffmanr   r   r   r   r6   ra   r   r    r   r   <module>r      s3      "  
7>/d4@nIr   