
    
9j)y                      S SK Jr  S SKrS SKrS SKrS SKr	S SK
Jr  S SK
Jr  S SKJr  S SKJr  S SKJr  S SKJr  S S	KJr  S SKr " S
 S5      rS rS rSzS jrS r S r!S r"S r#0 SS_SS_SS_SS_SS_SS_SS_SS_SS_SS S!S".S4_S#S$S!S".S4_S%S_S&S_S'S(_S)S(_S*S(_S+S(_0 S,S(_S-S_S.S_S/S0_S1S0_S2S(_S3S(_S4S(_S5S6_S7S(_S8S9_S:S9_S;S<_S=S>_S?S@_SASB_Er$0 SSC_S\RJ                  S4_SSC_SSC_SSC_SSC_SSC_SSC_SSC_SSD_S#SD_S%SC_S&SC_S'SC_S)SC_S*SC_S+SC_0 S,SC_S-SC_S.SC_S/\RJ                  S4_S1\RJ                  S4_S2SC_S3SC_S4SC_S5SC_S7SC_S8SD_S:SD_S;\RJ                  S4_S=SE_S?\RJ                  S4_SA\RJ                  S4_Er&SF r'\RP                  " 5       SG 5       r)S{SH jr*S|SI jr+SzSJ jr,S}SK jr-S|SL jr.S~SM jr/SSN jr0SO r1SSP jr2SSQ jr3SSR jr4SSS jr5SzST jr6SzSU jr7SV r8SW r9SSX jr:SY r;SZ r<S[ r=S\ r>S] r?S^ r@S_ rAS` rBSa rCSb rDSc rESd rFSe rG " Sf Sg5      rH " Sh Si\H5      rI " Sj Sk\H5      rJ " Sl Sm\H5      rK " Sn So\H5      rLS|Sp jrMSqrNSSr jrO  SSs jrPSSt jrQSSu jrRSzSv jrSSSw jrTSx rUSSy jrVg)    )annotationsN)driver)runtime)cusparse)_dtype)device)stream)_utilc                  h    \ rS rSrS r\S 5       rS r\R                  4S jr
S rS rS rS	 rS
rg)MatDescriptor   c                    Xl         g N
descriptor)selfr   s     H/home/wildlama/miniconda3/lib/python3.13/site-packages/cupyx/cusparse.py__init__MatDescriptor.__init__   s    $    c                B    [         R                  " 5       n[        U5      $ r   )	_cusparsecreateMatDescrr   )clsdescrs     r   createMatDescriptor.create   s    ((*U##r   c                    U R                   S4$ )N )r   )r   s    r   
__reduce__MatDescriptor.__reduce__   s    {{Br   c                    U" 5       (       a  g U R                   (       a(  [        R                  " U R                   5        S U l         g g r   )r   r   destroyMatDescrr   is_shutting_downs     r   __del__MatDescriptor.__del__    s3    ??%%doo6"DO r   c                F    [         R                  " U R                  U5        g r   )r   
setMatTyper   )r   typs     r   set_mat_typeMatDescriptor.set_mat_type'   s    T__c2r   c                F    [         R                  " U R                  U5        g r   )r   setMatIndexBaser   )r   bases     r   set_mat_index_base MatDescriptor.set_mat_index_base*   s    !!$//48r   c                F    [         R                  " U R                  U5        g r   )r   setMatFillModer   )r   	fill_modes     r   set_mat_fill_modeMatDescriptor.set_mat_fill_mode-         )<r   c                F    [         R                  " U R                  U5        g r   )r   setMatDiagTyper   )r   	diag_types     r   set_mat_diag_typeMatDescriptor.set_mat_diag_type0   r7   r   r   N)__name__
__module____qualname____firstlineno__r   classmethodr   r    r
   r%   r&   r+   r0   r5   r;   __static_attributes__r   r   r   r   r      sD    % $ $ (-'='= #39==r   r   c                    U  Vs/ s H  oc  M  UR                   PM     nn[        R                  " [        R                  U5      nU  Vs/ s H)  nUb!  UR                   U:w  a  UR                  U5      OUPM+     sn$ s  snf s  snf r   )dtype
_functoolsreduce_numpypromote_typesastype)xsxdtypesrD   s       r   _cast_common_typerM   4   sz    !3r!gaggrF3f22F;E  !}E1AAHHUOqH  4s   A=A=
0Bc                    U R                   R                  [        R                  :X  a  [	        SR                  X5      5      eg)z*Raise ValueError if *a* has int64 indices.z={} does not support int64 indices (cuSPARSE {} is int32-only)N)indicesrD   _cupyint64
ValueErrorformat)a	func_names     r   _check_int32_indicesrV   ;   s:    yy%++%**0&+&' 	' &r   c                   Uc  U R                   nU R                  S   S-
  nUS:  a\  [        U S   5      nUS[        US5      -  :  a;  [        R
                  " X1S9n[        R                  " U SS USS	9R                  US
S9$ [        R                  " [        R
                  " X!S9[        R                  " U 5      5      $ )am  Expand compressed ``indptr`` to per-nnz major-axis indices.

Inverse of :func:`_build_indptr`.  ``dtype`` defaults to ``indptr.dtype``.

For most matrices the ``cupy.repeat(arange(major), diff)`` formula
is the right call: O(major + nnz) memory, no host sync.  But when
the major axis dwarfs ``nnz`` (e.g. the (2, 2**31+5) CSC produced
by transposing a wide CSR with a single stored entry), the
``arange(major)`` allocation dominates -- 17 GB of intermediate
state for one nnz.  In that regime fall back to a searchsorted
formula that uses O(nnz log major) memory at the cost of one D2H
sync to read ``int(indptr[-1])``.
Nr      i @     rD   right)sideFcopy)
rD   shapeintmaxrP   arangesearchsortedrI   repeatdiff)indptrrD   nrowsnnzrc   s        r   _indptr_to_coorj   D   s     }LLOaE &*o1s3{?"\\#3F%%qr
F228&U&2KL<<U(%**V*<> >r   c                    [         R                  " US-   US9n[         R                  R                  USS U S5        [         R                  " X3S9  U$ )zBuild compressed indptr from per-nnz major-axis assignments.

Mirrors the histogram + prefix-sum recipe used by scipy.sparse,
works with both int32 and int64 ``dtype``.
rX   r[   Nout)rP   zerosaddatcumsum)row_indicesn_rowsrD   rg   s       r   _build_indptrrt   e   sC     [[!51F	IILL[!,	LL$Mr   c                .   U R                   R                  U:X  a  U $ U R                  R                  U R                  U R
                  R                  U5      U R                   R                  U5      U R                  [        U SS5      [        U SS5      S9$ )zReturn a CSR/CSC matrix with indices and indptr cast to *dtype*.

Data array is shared (no copy).  Used to promote int32 index arrays to
int64 before calling a cuSPARSE function that requires uniform int64.
_has_canonical_formatN_has_sorted_indiceshas_canonical_formathas_sorted_indices)	rg   rD   	__class___from_partsdatarO   rI   r`   getattr)mrD   s     r   _with_indices_dtyper   q   s     	xx~~ ;;""			  ')?	$&."$d, # - -r   c                P    U (       a  [         R                  $ [         R                  $ r   )r   CUSPARSE_OPERATION_TRANSPOSE CUSPARSE_OPERATION_NON_TRANSPOSE)transs    r   _transpose_flagr      s    555999r   c                    US:X  a  SnO!US:X  a  SnOUS:X  a  SnOUS:X  a  SnO[         e[        [        X0-   5      nU" U6 $ )NfsdFcDz)	TypeErrorr~   r   )namerD   argsprefixr   s        r   _call_cusparser      sN    |	#	#	#	6=)Ad8Or   csrmv)@  *  csrmvExcsrmmcsrmm2csrgeamcsrgeam2)<#  Ncsrgemmcsrgemm2)r   .  gthrspmv'  r   )LinuxWindowsspmmi=(  	csr2dense	csc2densecsrsort)r   Ncscsortcoosortcoo2csrcsr2coocsr2csccsc2csr
csr2cscEx2)r   N
csc2csrEx2	dense2csc	dense2csrcsr2csr_compresscsrsm2)r   r   csrilu02denseToSparse)i$,  NsparseToDensespgemm)i\+  Nspsm)iP-  Nspgeam)i Nspgemm_int64)2  N)i1  N)  N)iNc                    [        U [        5      (       a:  [        R                  " 5       nX;  a  SR	                  U5      n[        U5      eX   $ U $ )Nz/No version information specified for the OS: {})
isinstancedict	_platformsystemrS   rR   )rK   os_namemsgs      r   _get_avail_version_from_specr      sM    !T""$CJJCS/!zHr   c                :   [         R                  (       d  [        n[        R                  " 5       nO[
        n[        R                  " 5       nX;  a  SR                  U 5      n[        U5      eX   u  pE[        U5      n[        U5      nUb  X$:  a  gUb  X%:  a  gg)Nz1No available version information specified for {}FT)
_runtimeis_hip_available_cusparse_versionr   get_build_version_available_hipsparse_version_driverrS   rR   r   )r   available_versionversionr   version_addedversion_removeds         r   check_availabilityr     s    ??7--/8++-$AHHNo%6%<"M0?M2?CO W%<"w'Ar   c                 T    [         R                  " [        R                  " 5       5      $ r   )r   
getVersion_deviceget_cusparse_handler   r   r   r   r     s     ; ; =>>r   c                   [        S5      (       d  [        S5      eUb&  UR                  R                  (       d  [	        S5      eU(       d  U R
                  OU R
                  SSS2   nUS   [        U5      :w  a  [	        S5      e[        R                  " 5       nUu  p[        XU5      u  pnU R                  n
Uc  [        R                  " X5      n[        R                  " X:5      R                  n[        R                  " XJ5      R                  n[!        SU
U[#        U5      U R
                  S   U R
                  S   U R$                  UR&                  U R(                  R*                  U R&                  R&                  R,                  U R.                  R&                  R,                  U R0                  R&                  R,                  UR&                  R,                  UR&                  UR&                  R,                  5        U$ )	a$  Matrix-vector product for a CSR-matrix and a dense vector.

.. math::

   y = \alpha * o_a(A) x + \beta y,

where :math:`o_a` is a transpose function when ``transa`` is ``True`` and
is an identity function otherwise.

Args:
    a (cupyx.cusparse.csr_matrix): Matrix A.
    x (cupy.ndarray): Vector x.
    y (cupy.ndarray or None): Vector y. It must be F-contiguous.
    alpha (float): Coefficient for x.
    beta (float): Coefficient for y.
    transa (bool): If ``True``, transpose of ``A`` is used.

Returns:
    cupy.ndarray: Calculated ``y``.

r   zcsrmv is not available.N!expected F-contiguous array for yrY   rX   dimension mismatchr   )r   RuntimeErrorflagsf_contiguousrR   r`   lenr   r   rM   rD   rP   rn   rG   arrayctypesr   r   ri   r}   _descrr   ptrrg   rO   )rT   rK   yalphabetatransaa_shapehandler   nrD   s              r   r   r      sk   , g&&455}QWW11<==#agg2GqzSV-..((*FDAa(GA!GGEyKK!LL&--E<<$++D'	
AGGAJuzz1883F3F	**AIINN,>,>	

DIIqvvzz+ Hr   c                r   U R                   R                   R                  S-  S:w  a  gU R                  R                   R                  S-  S:w  a  gU R                  R                   R                  S-  S:w  a  gUR                   R                  S-  S:w  a  gUb  UR                   R                  S-  S:w  a  gg)am  Check if the pointers of arguments for csrmvEx are aligned or not

Args:
    a (cupyx.cusparse.csr_matrix): Matrix A.
    x (cupy.ndarray): Vector x.
    y (cupy.ndarray or None): Vector y.

    Check if a, x, y pointers are aligned by 128 bytes as
    required by csrmvEx.

Returns:
    bool:
    ``True`` if all pointers are aligned.
    ``False`` if otherwise.

   r   FT)r}   r   rg   rO   )rT   rK   r   s      r   csrmvExIsAlignedr   S  s    $ 	vv{{!xx}}3!#yy~~C1$vvzzC1}c)Q.r   c                0   [        S5      (       d  [        S5      eUb&  UR                  R                  (       d  [	        S5      eU R
                  S   [        U5      :w  a  [	        S5      e[        R                  " 5       nU R
                  u  px[        XU5      u  pnU R                  n	Uc  [        R                  " Xy5      n[        R                  " U	5      n
U(       a  [        R                   O[        R"                  n[        R$                  n[&        R(                  " X95      R*                  n[&        R(                  " XI5      R*                  n[-        XU5      (       d   e[        R.                  " XkUU R
                  S   U R
                  S   U R0                  UR2                  U
U R4                  R6                  U R2                  R2                  R8                  U
U R:                  R2                  R8                  U R<                  R2                  R8                  UR2                  R8                  XR2                  U
UR2                  R8                  X5      n[        R>                  " US5      nUR2                  R8                  S-  S:X  d   e[        R@                  " XkUU R
                  S   U R
                  S   U R0                  UR2                  U
U R4                  R6                  U R2                  R2                  R8                  U
U R:                  R2                  R8                  U R<                  R2                  R8                  UR2                  R8                  XR2                  U
UR2                  R8                  XUR2                  R8                  5        U$ )	a  Matrix-vector product for a CSR-matrix and a dense vector.

.. math::

   y = \alpha * A x + \beta y,

Args:
    a (cupyx.cusparse.csr_matrix): Matrix A.
    x (cupy.ndarray): Vector x.
    y (cupy.ndarray or None): Vector y. It must be F-contiguous.
    alpha (float): Coefficient for x.
    beta (float): Coefficient for y.
    merge_path (bool): If ``True``, merge path algorithm is used.

    All pointers must be aligned with 128 bytes.

Returns:
    cupy.ndarray: Calculated ``y``.

r   zcsrmvEx is not available.r   rX   r   r   br   )!r   r   r   r   rR   r`   r   r   r   rM   rD   rP   rn   r   to_cuda_dtyper   CUSPARSE_ALG_MERGE_PATHCUSPARSE_ALG_NAIVEr   rG   r   r   r   csrmvEx_bufferSizeri   r}   r   r   r   rg   rO   emptyr   )rT   rK   r   r   r   
merge_pathr   r   r   rD   datatypealgmodetransa_flag
bufferSizebufs                  r   r   r   r  sw   * i((677}QWW11<==wwqzSV-..((*F77DAa(GA!GGEyKK!##E*H //!44 <<KLL&--E<<$++DA!$$$$--	
AGGAJuzz8	QVV[[__h	199>>--	

Hii	

H(J ++j#
&C88<<#"""	
AGGAJuzz8	QVV[[__h	199>>--	

Hii	

H6 Hr   c                   [        S5      (       d  [        S5      eU R                  S:w  d  UR                  S:w  a  [        S5      eUR                  R
                  (       d  [        S5      eUb&  UR                  R
                  (       d  [        S5      eU(       d  U R                  OU R                  SSS2   nUS	   UR                  S
   :w  a  [        S5      e[        R                  " 5       nUu  pUR                  S	   n
[        XU5      u  pnUc#  [        R                  " X4U R                  S5      nU	nUn[        R                  " X0R                  5      R                  n[        R                  " X@R                  5      R                  n[!        SU R                  U[#        U5      U R                  S
   XR                  S	   U R$                  UR&                  U R(                  R*                  U R&                  R&                  R,                  U R.                  R&                  R,                  U R0                  R&                  R,                  UR&                  R,                  XR&                  UR&                  R,                  U5        U$ )aD  Matrix-matrix product for a CSR-matrix and a dense matrix.

.. math::

   C = \alpha o_a(A) B + \beta C,

where :math:`o_a` is a transpose function when ``transa`` is ``True`` and
is an identity function otherwise.

Args:
    a (cupyx.scipy.sparse.csr): Sparse matrix A.
    b (cupy.ndarray): Dense matrix B. It must be F-contiguous.
    c (cupy.ndarray or None): Dense matrix C. It must be F-contiguous.
    alpha (float): Coefficient for AB.
    beta (float): Coefficient for C.
    transa (bool): If ``True``, transpose of A is used.

Returns:
    cupy.ndarray: Calculated C.

r   zcsrmm is not available.   expected 2-D matrices!expected F-contiguous array for bN!expected F-contiguous array for crY   rX   r   r   r   )r   r   ndimrR   r   r   r`   r   r   rM   rP   rn   rD   rG   r   r   r   r   ri   r}   r   r   r   rg   rO   )rT   r   r   r   r   r   r   r   r   kr   ldbldcs                r   r   r     s   , g&&455vv{affk01177<==}QWW11<==#agg2GqzQWWQZ-..((*FDA	
Aa(GA!yKK-
C
CLL(//E<<gg&--D'	
Awwqz155

AHH''	199>>--	

CAFFJJ5 Hr   c                   [        S5      (       d  [        S5      eU R                  S:w  d  UR                  S:w  a  [        S5      eU R                  (       d  [        S5      eUR
                  R                  (       d  [        S5      eUb&  UR
                  R                  (       d  [        S5      eU(       a	  U(       a   eU(       d  U R                  OU R                  SSS	2   nU(       d  UR                  OUR                  SSS	2   nUS
   US   :w  a  [        S5      e[        R                  " 5       n	Uu  pUS
   n[        XU5      u  pnUc#  [        R                  " X4U R                  S5      nUR                  S   nUR                  S   n[        U5      n[        U5      n[        R                   " X0R                  5      R"                  n[        R                   " X@R                  5      R"                  n[%        SU R                  XUU R                  S   XR                  S
   U R&                  UR(                  U R*                  R,                  U R(                  R(                  R.                  U R0                  R(                  R.                  U R2                  R(                  R.                  UR(                  R.                  XR(                  UR(                  R.                  U5        U$ )a  Matrix-matrix product for a CSR-matrix and a dense matrix.

.. math::

   C = \alpha o_a(A) o_b(B) + \beta C,

where :math:`o_a` and :math:`o_b` are transpose functions when ``transa``
and ``tranb`` are ``True`` respectively. And they are identity functions
otherwise.
It is forbidden that both ``transa`` and ``transb`` are ``True`` in
cuSPARSE specification.

Args:
    a (cupyx.scipy.sparse.csr): Sparse matrix A.
    b (cupy.ndarray): Dense matrix B. It must be F-contiguous.
    c (cupy.ndarray or None): Dense matrix C. It must be F-contiguous.
    alpha (float): Coefficient for AB.
    beta (float): Coefficient for C.
    transa (bool): If ``True``, transpose of A is used.
    transb (bool): If ``True``, transpose of B is used.

Returns:
    cupy.ndarray: Calculated C.

r   zcsrmm2 is not available.r   r   expected canonical format for ar   Nr   rY   rX   r   r   r   )r   r   r   rR   ry   r   r   r`   r   r   rM   rP   rn   rD   r   rG   r   r   r   ri   r}   r   r   r   rg   rO   )rT   r   r   r   r   r   transbr   b_shaper   r   r   r   r   r   op_aop_bs                    r   r   r     s   4 h''566vv{affk011!!:;;77<==}QWW11<==6""#agg2G#agg2GqzWQZ-..((*FDA
Aa(GA!yKK-
''!*C
''!*C6"D6"DLL(//E<<gg&--D!''dAGGAJ771:quu

AHH''	199>>--	

CAFFJJ5 Hr   c                V
   [        S5      (       d  [        S5      e[        U [        R                  R
                  R                  5      (       d#  [        SR                  [        U 5      5      5      e[        U[        R                  R
                  R                  5      (       d#  [        SR                  [        U5      5      5      e[        U S5        [        US5        U R                  (       d  [        S5      eUR                  (       d  [        S5      eU R                  UR                  :w  a  [        S5      e[        R                  " 5       nU R                  u  pV[!        X5      u  p["        R$                  " SS5      n[&        R(                  " U[&        R*                  5        [,        R/                  5       n[0        R$                  " US	-   S5      n	[&        R2                  " XEUU R4                  R6                  U R8                  U R:                  R<                  R>                  U R@                  R<                  R>                  UR4                  R6                  UR8                  UR:                  R<                  R>                  UR@                  R<                  R>                  UR6                  U	R<                  R>                  URB                  R<                  5        [0        R$                  " [E        U5      S5      n
[0        R$                  " [E        U5      U RF                  5      n["        RH                  " X RF                  5      RB                  n["        RH                  " X0RF                  5      RB                  n[K        SU RF                  XEXbR<                  U R4                  R6                  U R8                  U R<                  R<                  R>                  U R:                  R<                  R>                  U R@                  R<                  R>                  UR<                  UR4                  R6                  UR8                  UR<                  R<                  R>                  UR:                  R<                  R>                  UR@                  R<                  R>                  UR6                  UR<                  R>                  U	R<                  R>                  U
R<                  R>                  5        [        R                  R
                  R                  XU	4U R                  S
9nSUl&        U$ )=  Matrix-matrix addition.

.. math::
    C = \alpha A + \beta B

Args:
    a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
    b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
    alpha (float): Coefficient for A.
    beta (float): Coefficient for B.

Returns:
    cupyx.scipy.sparse.csr_matrix: Result matrix.

r   zcsrgeam is not available.unsupported type (actual: {})r  expected canonical format for binconsistent shapesr   irX   r`   T)'r   r   r   cupyxscipysparse
csr_matrixr   rS   typerV   ry   rR   r`   r   r   rM   rG   r   r   setPointerModeCUSPARSE_POINTER_MODE_HOSTr   r   rP   xcsrgeamNnzr   r   ri   rg   r}   r   rO   r   ra   rD   r   r   rv   )rT   r   r   r   r   r   r   ri   c_descrc_indptr	c_indicesc_datar   s                r   r   r   5  sZ     i((677a++66777>>tAwGHHa++66777>>tAwGHHI&I&!!:;;!!:;;ww!''.//((*F77DAQ"DA
,,r3
C	446 ""$G{{1q5#&H1	QUUAHHMM$5$5qyy~~7I7I	QUUAHHMM$5$5qyy~~7I7IHMM--szz	@ CHc*I[[S177+FLL(//E<<gg&--D1771jj	QUUAFFKKOO	199>>--tyy	QUUAFFKKOO	199>>--FKKOOX]]->-> 	%%	H%QWW 	& 	6A"AHr   c                >   [         R                  " U R                  R                  UR                  R                  5      nUS:w  a(  U R                  U R                  R                  U5      -  OU R                  nUS:w  a(  UR                  UR                  R                  U5      -  OUR                  n[        U R                  U5      n[        UR                  U5      n[        R                  " Xx/5      n	[        R                  " U R                  UR                  /5      n
[        R                  " XV/5      n[        R                  R                  R                  R                  XXR                  5      nSUl        UR#                  5         UR%                  5       $ )zsPure-CuPy CSR addition for int64: C = alpha*A + beta*B.

Uses COO concatenation + sum_duplicates.  O(nnz log nnz).
rX   F)rG   result_typerO   rD   r}   r  rj   rg   rP   concatenater  r  r  
coo_matrixr|   r`   ry   sum_duplicatestocsr)rT   r   r   r   	idx_dtypea_datab_dataa_rowsb_rowsrowscolsr}   coos                r   _cupy_csrgeam_int64r'  y  s"    ""199??AIIOODI .3aZQVVaggll5))QVVF,0AIQVVaggll4((166FAHHi0FAHHi0Ff-.Daii34Df-.D
++


'
'
3
3D''#C$C99;r   c                `   [        U [        R                  R                  R                  5      (       d#  [        SR                  [        U 5      5      5      e[        U[        R                  R                  R                  5      (       d#  [        SR                  [        U5      5      5      eU R                  R                  [        R                  :X  d(  UR                  R                  [        R                  :X  a5  [        S5      (       a  [        XX#5      $ [        X5      u  p[        XX#5      $ [        S5      (       d  [!        S5      eU R"                  (       d  [%        S5      eUR"                  (       d  [%        S5      eU R&                  UR&                  :w  a  [%        S5      e[(        R*                  " 5       nU R&                  u  pV[        X5      u  p[,        R.                  " SS	5      n[0        R2                  " U[0        R4                  5        [,        R6                  " X R                  5      R8                  n[,        R6                  " X0R                  5      R8                  n[:        R=                  5       n[        R.                  " US
-   S	5      n	Sn
[?        SU R                  XEXbR@                  U RB                  RD                  U RF                  U R@                  R@                  RH                  U RJ                  R@                  RH                  U R                  R@                  RH                  UR@                  URB                  RD                  URF                  UR@                  R@                  RH                  URJ                  R@                  RH                  UR                  R@                  RH                  URD                  XR@                  RH                  U
5      n[        R.                  " U[,        RL                  5      n[0        RN                  " XEX`RB                  RD                  U RF                  U RJ                  R@                  RH                  U R                  R@                  RH                  URB                  RD                  URF                  URJ                  R@                  RH                  UR                  R@                  RH                  URD                  U	R@                  RH                  UR8                  R@                  UR@                  RH                  5        [        R.                  " [Q        U5      S	5      n[        R.                  " [Q        U5      U R                  5      n[?        SU R                  XEXbR@                  U RB                  RD                  U RF                  U R@                  R@                  RH                  U RJ                  R@                  RH                  U R                  R@                  RH                  UR@                  URB                  RD                  URF                  UR@                  R@                  RH                  URJ                  R@                  RH                  UR                  R@                  RH                  URD                  UR@                  RH                  U	R@                  RH                  UR@                  RH                  UR@                  RH                  5        [        R                  R                  R	                  XU	4U R&                  S9nSUl)        U$ )r  r  r   r   zcsrgeam2 is not available.r  r	  r
  r   r  rX   r   csrgeam2_bufferSizeExtr  T)*r   r  r  r  r  r   rS   r  rO   rD   rP   rQ   r   r   rM   r'  r   ry   rR   r`   r   r   rG   r   r   r  r  r   r   r   r   r   r}   r   r   ri   r   rg   int8xcsrgeam2Nnzra   rv   )rT   r   r   r   r   r   r   ri   r  r  null_ptr	buff_sizebuffr  r  r   s                   r   r   r     s     a++66777>>tAwGHHa++66777>>tAwGHH
 	yy%++%EKK)Gh''!,, &"155j))788!!:;;!!:;;ww!''.//((*F77DAQ"DA
,,r3
C	446 LL(//E<<gg&--D""$G{{1q5#&HH !''1jj!(("5"5quuaffkkoo	199>>--tyy!((:M:M	qvv{{ 1 1199>>3E3EHmm&7&7CI ;;y&++.D1hh))155!((--2C2C			AHH//8I8I			G..0A0A

	(
 CHc*I[[S177+FAGG1jj!(("5"5quuaffkkoo	199>>--tyy!((:M:M	qvv{{ 1 1199>>3E3EFKKOOX]]->->DIIMM+ 	%%	H%QWW 	& 	6A"AHr   c                $   [        S5      (       du  U R                  R                  [        R                  :X  d(  UR                  R                  [        R                  :X  a  [        X5      u  p[        XX#5      $ [        XX#5      $ [        U [        R                  R                  R                  5      (       d#  [        SR                  [        U 5      5      5      e[        U[        R                  R                  R                  5      (       d#  [        SR                  [        U5      5      5      eU R                   (       d  [#        S5      eUR                   (       d  [#        S5      eU R$                  UR$                  :w  a  [#        S5      e[&        R(                  " U R                  R                  UR                  R                  5      n[        X5      u  p[+        X5      n [+        X5      nU R$                  u  pV[,        R.                  " 5       n[        R0                  " US-   US9n[        R                  R                  R                  R3                  [        R0                  " SU R                  5      [        R0                  " SU5      XU45      n	[4        R7                  U 5      n
[4        R7                  U5      n[4        R7                  U	5      n[&        R8                  " X R                  S9n[&        R8                  " X1R                  S9n[:        R<                  " U R                  5      n[>        R@                  n[>        RB                  n[>        RD                  " 5       n [>        RF                  " UUUURH                  RJ                  U
RL                  URH                  RJ                  URL                  URL                  UUU5      n[        R0                  " U[        RN                  5      n[>        RP                  " UUUURH                  RJ                  U
RL                  URH                  RJ                  URL                  URL                  UUUURJ                  RR                  5        [&        R8                  " SS	S9n[&        R8                  " SS	S9n[&        R8                  " SS	S9n[>        RT                  " URL                  URH                  RJ                  URH                  RJ                  URH                  RJ                  5        [W        U5      n[        R0                  " UU5      n[        R0                  " UU R                  5      n[>        RX                  " URL                  URJ                  RR                  URJ                  RR                  URJ                  RR                  5        [>        RZ                  " UUUURH                  RJ                  U
RL                  URH                  RJ                  URL                  URL                  UUUURJ                  RR                  5        [>        R\                  " U5        [        R                  R                  R                  R3                  UUXU4S
S9n	U	$ ! [>        R\                  " U5        f = f)aR  Sparse matrix addition using the Generic API: C = alpha*A + beta*B.

Uses ``cusparseSpGEAM`` when available.  Not yet in any public CUDA
release as of 13.2, but present in dev and verified working
(~2x faster than csrgeam2 for int32, supports int64 natively).
Falls back to ``_cupy_csrgeam_int64`` for int64 or ``csrgeam2``
for int32.

Args:
    a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
    b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
    alpha (scalar): Coefficient for A.
    beta (scalar): Coefficient for B.

Returns:
    cupyx.scipy.sparse.csr_matrix: Result matrix C.

r   r  r  r	  r
  rX   r[   r   rQ   Try   )/r   rO   rD   rP   rQ   rM   r'  r   r   r  r  r  r  r   rS   r  ry   rR   r`   rG   r  r   r   r   r   r|   SpMatDescriptorr   r   r   r   r   CUSPARSE_SPGEAM_ALG_DEFAULTr   spGEAM_createDescrspGEAM_bufferSizer   r}   descr*  
spGEAM_nnzr   spMatGetSizera   csrSetPointersspGEAMspGEAM_destroyDescr)rT   r   r   r   r  r   r   r   r  r   mat_amat_bmat_c	alpha_arrbeta_arr
cuda_dtypealgopr5  buf_sizer   
c_num_rows
c_num_cols	c_nnz_arrc_nnzr  r  s                              r   r   r     s   & h'' 99??ekk)QYY__-K$Q*DA&qU99e**a++66777>>tAwGHHa++66777>>tAwGHH!!:;;!!:;;ww!''.//""199??AIIOODIQ"DA 	A)AA)A77DA((*F {{1q5	2H%%11AqwwQ	!:a&	A ""1%E""1%E""1%EU''2I||D0H%%agg.J

/
/C		3	3B'')D$,..B!!5::OO  %**JJ
C	/
 kk(EJJ/ 	B!!5::OO  %**JJ
Csxx||		= \\!73
\\!73
LL'2	uzz:+<+<+A+A)0055y7G7G7L7L	NIKKy1	UAGG,  X]]->->!*!3!3V[[__	F 	B!!5::OO  %**JJ
Csxx||		= 	%%d+%%11	8V! 	2 	#A H 	%%d+s   (KY7 7Zc                	   [        S5      (       d  [        S5      eU R                  S:w  d  UR                  S:w  a  [        S5      e[	        U S5        [	        US5        U R
                  (       d  [        S5      eUR
                  (       d  [        S5      eU(       d  U R                  OU R                  SSS2   nU(       d  UR                  OUR                  SSS2   nUS	   US
   :w  a  [        S5      e[        R                  " 5       nUu  pxUS	   n	[        X5      u  pU R                  S
:X  d  UR                  S
:X  a3  [        R                  R                  R                  Xy4U R                  S9$ [!        U5      n
[!        U5      n["        R$                  " SS5      n[&        R(                  " U[&        R*                  5        [,        R/                  5       n[0        R$                  " US	-   S5      n[&        R2                  " XjXXU R4                  R6                  U R                  U R8                  R:                  R<                  U R>                  R:                  R<                  UR4                  R6                  UR                  UR8                  R:                  R<                  UR>                  R:                  R<                  UR6                  UR:                  R<                  UR@                  R:                  5        [0        R$                  " [C        U5      S5      n[0        R$                  " [C        U5      U R                  5      n[E        SU R                  XjXXU R4                  R6                  U R                  U R:                  R:                  R<                  U R8                  R:                  R<                  U R>                  R:                  R<                  UR4                  R6                  UR                  UR:                  R:                  R<                  UR8                  R:                  R<                  UR>                  R:                  R<                  UR6                  UR:                  R<                  UR:                  R<                  UR:                  R<                  5        [        R                  R                  R                  UX4Xy4S9nSUl#        U$ )ae  Matrix-matrix product for CSR-matrix.

math::
   C = op(A) op(B),

Args:
    a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
    b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
    transa (bool): If ``True``, transpose of A is used.
    transb (bool): If ``True``, transpose of B is used.

Returns:
    cupyx.scipy.sparse.csr_matrix: Calculated C.

r   zcsrgemm is not available.r   r   r  r	  NrY   rX   r   r   r[   r   r  r  T)$r   r   r   rR   rV   ry   r`   r   r   rM   ri   r  r  r  r  rD   r   rG   r   r   r  r  r   r   rP   xcsrgemmNnzr   r   rg   r}   r   rO   r   ra   r   rv   )rT   r   r   r  r   r  r   r   r   r   r  r  ri   r  r  r  r  r   s                     r   r   r   T  sW     i((677vv{affk011I&I&!!:;;!!:;;#agg2G#agg2GqzWQZ-..((*FDA
AQ"DAuuzQUUaZ{{!!,,aV177,CC6"D6"D
,,r3
C	446 ""$G{{1q5#&HdqQXX%8%8!%%	199>>--qxx/B/BAEE	199>>--w/A/A3::??	, CHc*I[[S177+F177dqQXX%8%8!%%	**AIINN,>,>	QUUAFFKKOOQXX]]5F5F			FKKOOX]]->-> 	%%	%aV 	& 	5A"AHr   c                t   [        S5      (       d  [        S5      eU R                  S:w  d  UR                  S:w  a  [        S5      e[	        U [
        R                  R                  R                  5      (       d#  [        SR                  [        U 5      5      5      e[	        U[
        R                  R                  R                  5      (       d#  [        SR                  [        U5      5      5      e[        U S5        [        US5        U R                  (       d  [        S5      eUR                  (       d  [        S5      eU R                  S   UR                  S	   :w  a  [        S
5      eUGb  UR                  S:w  a  [        S5      e[	        U[
        R                  R                  R                  5      (       d#  [        SR                  [        U5      5      5      e[        US5        UR                  (       d  [        S5      eU R                  S	   UR                  S	   :w  d   UR                  S   UR                  S   :w  a  [        S
5      e[        R                   (       a$  ["        R$                  " 5       S:  a  [        S5      e[&        R(                  " 5       nU R                  u  pgUR                  u  pUc  [+        X5      u  pO[+        XU5      u  pn[,        R.                  " 5       n
[0        R2                  " X0R4                  5      R6                  nS	nUc  Un[8        R;                  5       nS	nUnUnUnO[0        R2                  " X@R4                  5      R6                  nUR<                  nUR>                  nUR@                  nUR<                  R<                  RB                  nURD                  R<                  RB                  nURF                  R<                  RB                  n[I        SU R4                  XVXUR<                  U R>                  RJ                  U R@                  U RD                  R<                  RB                  U RF                  R<                  RB                  UR>                  RJ                  UR@                  URD                  R<                  RB                  URF                  R<                  RB                  XRJ                  UUUU
5      n[L        RN                  " U[0        RP                  5      n[0        RN                  " SS5      n[,        RR                  " U[,        RT                  5        [8        R;                  5       n[L        RN                  " US-   S5      n[,        RV                  " XVXU R>                  RJ                  U R@                  U RD                  R<                  RB                  U RF                  R<                  RB                  UR>                  RJ                  UR@                  URD                  R<                  RB                  URF                  R<                  RB                  URJ                  UUUURJ                  UR<                  RB                  UR6                  R<                  U
UR<                  RB                  5        [L        RN                  " [Y        U5      S5      n[L        RN                  " [Y        U5      U R4                  5      n[I        SU R4                  XVXUR<                  U R>                  RJ                  U R@                  U R<                  R<                  RB                  U RD                  R<                  RB                  U RF                  R<                  RB                  UR>                  RJ                  UR@                  UR<                  R<                  RB                  URD                  R<                  RB                  URF                  R<                  RB                  XRJ                  XUUURJ                  UR<                  RB                  UR<                  RB                  UR<                  RB                  U
UR<                  RB                  5        [
        R                  R                  R                  UUU4Xi4S9nSUl-        [,        R\                  " U
5        U$ )aq  Matrix-matrix product for CSR-matrix.

math::
   C = alpha * A * B + beta * D

Args:
    a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
    b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
    d (cupyx.scipy.sparse.csr_matrix or None): Sparse matrix D.
    alpha (scalar): Coefficient
    beta (scalar): Coefficient

Returns:
    cupyx.scipy.sparse.csr_matrix

r   zcsrgemm2 is not available.r   r   r  r  r	  rX   r   mismatched shapezexpected 2-D matrix for dzexpected canonical format for dr   z'd != None is supported since ROCm 4.2.0csrgemm2_bufferSizeExtr   r  r  T)/r   r   r   rR   r   r  r  r  r  r   rS   r  rV   ry   r`   r   r   r   r   r   r   rM   r   createCsrgemm2InforG   r   rD   r   r   r   r}   r   ri   r   rg   rO   r   r   rP   r   r*  r  r  xcsrgemm2Nnzra   rv   destroyCsrgemm2Info)rT   r   r   r   r   r   r   r   _r   infor,  	beta_datad_descrd_nnzd_datad_indptr	d_indicesr-  r.  rG  r  r  r  r  r   s                             r   r   r     s   " j))788vv{affk011a++66777>>tAwGHHa++66777>>tAwGHHJ'J'!!:;;!!:;;wwqzQWWQZ+,,}66Q;899!U[[//::;;;BB47KLLQ
+%%>??771:#qwwqzQWWQZ'?/00??w88:S@HII((*F77DA77DAy &1#A!,a'')DLL(//EHy	&&(	||D''*11II	((88==$$IINN&&	 !''1QXX%8%8!%%	199>>--qxx/B/BAEE	199>>--y:L:LxD*I ;;y&++.DLLS!EVY%I%IJ""$G{{1q5#&H1,,aeeQXX]]5F5F			AHH//8I8I			G..xHMM--u||/@/@$		 CJ,I[[UQWW-FAGG1QXX%8%8!%%	**AIINN,>,>	QUUAFFKKOOQXX]]5F5F			I'9'95)W//9>>--tTYY]]D 	%%	H%aV 	& 	5A"A!!$'Hr   c                   [        S5      (       d  [        S5      eU R                  nUR                  S;  a  [	        S5      eUc!  [
        R                  " U R                  USS9nO&UR                  R                  (       d  [	        S5      e[        R                  " 5       n[        SU R                  X0R                  S   U R                  S	   U R                  R                  U R                  R                  R                   U R"                  R                  R                   U R$                  R                  R                   UR                  R                   U R                  S   5        U$ )
a  Converts CSR-matrix to a dense matrix.

Args:
    x (cupyx.scipy.sparse.csr_matrix): A sparse matrix to convert.
    out (cupy.ndarray or None): A dense matrix to store the result.
        It must be F-contiguous.

Returns:
    cupy.ndarray: Converted result.

r   zcsr2dense is not available.fdFDunsupported dtyper   rD   order#expected F-contiguous array for outr   rX   )r   r   rD   charrR   rP   r   r`   r   r   r   r   r   r   r   r}   r   rg   rO   rK   rm   rD   r   s       r   r   r     s     k**899GGEzz,--
{kk!''c:yy%%BCC((*FQWW
AGGAJ(;(;	**AIINN,>,>aggaj	" Jr   c                   [        S5      (       d  [        S5      eU R                  nUR                  S;  a  [	        S5      eUc!  [
        R                  " U R                  USS9nO&UR                  R                  (       d  [	        S5      e[        R                  " 5       n[        SU R                  X0R                  S   U R                  S	   U R                  R                  U R                  R                  R                   U R"                  R                  R                   U R$                  R                  R                   UR                  R                   U R                  S   5        U$ )
a  Converts CSC-matrix to a dense matrix.

Args:
    x (cupyx.scipy.sparse.csc_matrix): A sparse matrix to convert.
    out (cupy.ndarray or None): A dense matrix to store the result.
        It must be F-contiguous.

Returns:
    cupy.ndarray: Converted result.

r   zcsc2dense is not available.rY  rZ  r   r[  r]  r   rX   )r   r   rD   r^  rR   rP   r   r`   r   r   r   r   r   r   r   r}   r   rO   rg   r_  s       r   r   r   1  s     k**899GGEzz,--
{kk!''c:yy%%BCC((*FQWW
AGGAJ(;(;	++QXX]]->->aggaj	" Jr   c                   U R                   nUS:X  a  gU R                  u  p#U R                  R                  [        R
                  :X  a  [        U R                  5      n[        R                  " [        R                  " U R                  U/5      5      nU R                  U   U R                  SS& U R                  U   U R                  SS& SU l        g[        S5      (       d  [        S5      e[        R                  " 5       n[         R"                  " XbX1U R                  R                  R$                  U R                  R                  R$                  5      n[        R&                  " US5      n[        R&                  " US5      n	U R                  R)                  5       n
[         R*                  " XaU	R                  R$                  5        [         R,                  " XbX1U R.                  R0                  U R                  R                  R$                  U R                  R                  R$                  U	R                  R$                  UR                  R$                  5	        [        S5      (       ap  [3        SU R                  XaU
R                  R$                  U R                  R                  R$                  U	R                  R$                  [         R4                  5        g[6        R9                  XR                  5      n[:        R9                  U
5      n[         R<                  " XlR>                  UR>                  5        g)	znSorts indices of CSR-matrix in place.

Args:
    x (cupyx.scipy.sparse.csr_matrix): A sparse matrix to sort.

r   NTr   zcsrsort is not available.r   r  r   ) ri   r`   rO   rD   rP   rQ   rj   rg   lexsortstackr}   rz   r   r   r   r   r   xcsrsort_bufferSizeExtr   r   r_   createIdentityPermutationxcsrsortr   r   r   CUSPARSE_INDEX_BASE_ZEROSpVecDescriptorr   DnVecDescriptorgatherr5  )rK   ri   r   r   rowr\  r   buffer_sizer   P	data_origdesc_xdesc_ys                r   r   r   S      %%C
ax77DAyy%++%QXX&ekk199c*:;<yy'		!FF5Mq	#i((677((*F221188==,,			K ++k3
'CCAI''QVVZZ@1188..0A0A			AFFJJ6 &!!AGG++QVV[[__FFJJ	::	<
 !''662 ''	2fkk:r   c                   U R                   nUS:X  a  gU R                  u  p#U R                  R                  [        R
                  :X  a  [        U R                  5      n[        R                  " [        R                  " U R                  U/5      5      nU R                  U   U R                  SS& U R                  U   U R                  SS& SU l        g[        S5      (       d  [        S5      e[        R                  " 5       n[         R"                  " XbX1U R                  R                  R$                  U R                  R                  R$                  5      n[        R&                  " US5      n[        R&                  " US5      n	U R                  R)                  5       n
[         R*                  " XaU	R                  R$                  5        [         R,                  " XbX1U R.                  R0                  U R                  R                  R$                  U R                  R                  R$                  U	R                  R$                  UR                  R$                  5	        [        S5      (       ap  [3        SU R                  XaU
R                  R$                  U R                  R                  R$                  U	R                  R$                  [         R4                  5        g[6        R9                  XR                  5      n[:        R9                  U
5      n[         R<                  " XlR>                  UR>                  5        g)	znSorts indices of CSC-matrix in place.

Args:
    x (cupyx.scipy.sparse.csc_matrix): A sparse matrix to sort.

r   NTr   zcscsort is not available.r   r  r   ) ri   r`   rO   rD   rP   rQ   rj   rg   rb  rc  r}   rz   r   r   r   r   r   xcscsort_bufferSizeExtr   r   r_   re  xcscsortr   r   r   rg  rh  r   ri  rj  r5  )rK   ri   r   r   colr\  r   rl  r   rm  rn  ro  rp  s                r   r   r     rq  r   c           
        U R                   nUS:X  a  gUS:X  a  U R                  (       a  gU R                  u  p4U R                  R                  [
        R                  :X  a  US:X  aA  [
        R                  " [
        R                  " U R                  U R                  /5      5      nORUS:X  aA  [
        R                  " [
        R                  " U R                  U R                  /5      5      nO[        S5      eU R                  U   U R                  SS& U R                  U   U R                  SS& U R                  U   U R                  SS& US:X  a  SU l        g[        S5      (       d  [        S5      e[        R                  " 5       n[         R"                  " XcXBU R                  R                  R$                  U R                  R                  R$                  5      n[
        R&                  " US	5      n[
        R&                  " US
5      n	U R                  R)                  5       n
[         R*                  " XbU	R                  R$                  5        US:X  a  [         R,                  " XcXBU R                  R                  R$                  U R                  R                  R$                  U	R                  R$                  UR                  R$                  5        OUS:X  a  [         R.                  " XcXBU R                  R                  R$                  U R                  R                  R$                  U	R                  R$                  UR                  R$                  5        O[        S5      eU R                  R0                  S:w  a  [        S5      (       ap  [3        SU R                  XbU
R                  R$                  U R                  R                  R$                  U	R                  R$                  [         R4                  5        O_[6        R9                  XR                  5      n[:        R9                  U
5      n[         R<                  " XlR>                  UR>                  5        US:X  a  SU l        gg)zSorts indices of COO-matrix in place.

Args:
    x (cupyx.scipy.sparse.coo_matrix): A sparse matrix to sort.
    sort_by (str): Sort the indices by row ('r', default) or column ('c').

r   Nrr   z!sort_by must be either 'r' or 'c'Fr   zcoosort is not available.r   r  ?r   ) ri   ry   r`   rk  rD   rP   rQ   rb  rc  ru  rR   r}   r   r   r   r   r   xcoosort_bufferSizeExtr   r   r_   re  xcoosortByRowxcoosortByColumnr^  r   rg  rh  r   ri  rj  r5  )rK   sort_byri   r   r   r\  r   rl  r   rm  rn  ro  rp  s                r   r   r     s    %%C
ax#~!0077DAuu{{ekk!c>MM%++quuaeen"=>E^MM%++quuaeen"=>E@AA55<a55<aFF5Mq	c>%*A"i((677((*F221155::>>155::>>;K
++k3
'CCAI''QVVZZ@#~qquuzz~~quuzz~~FFJJ	& 
C""qquuzz~~quuzz~~FFJJ	& <==ww||sf%%Y^^//

I>>@
 %++Avv6F$++I6FV[[&++>#~!& r   c                   [        U R                  S   5      nU R                  nU R                  R                  nUS:X  a  [
        R                  " US-   US9nOU[
        R                  :X  a  [        U R                  X5      nO[        R                  " 5       n[
        R                  " US-   US9n[        R                  " XPR                  R                  R                  X!UR                  R                  [        R                   5        ["        R$                  R&                  R(                  R+                  U R                  U R,                  X@R                  5      $ )Nr   rX   r[   )ra   r`   ri   rk  rD   rP   rn   rQ   rt   r   r   r   r   xcoo2csrr}   r   rg  r  r  r  r  r|   ru  )rK   r   ri   r  rg   r   s         r   r   r         AGGAJA
%%CI
axQU)4	ekk	!quua3,,.QU)4EEJJNNCKKOOY??	A ;;((44	vww( (r   c                   [        U R                  S   5      nU R                  nU R                  R                  nUS:X  a  [
        R                  " US-   US9nOU[
        R                  :X  a  [        U R                  X5      nO[        R                  " 5       n[
        R                  " US-   US9n[        R                  " XPR                  R                  R                  X!UR                  R                  [        R                   5        ["        R$                  R&                  R(                  R+                  U R                  U R,                  X@R                  5      $ )NrX   r   r[   )ra   r`   ri   ru  rD   rP   rn   rQ   rt   r   r   r   r   r~  r}   r   rg  r  r  r  
csc_matrixr|   rk  )rK   r   ri   r  rg   r   s         r   coo2cscr    r  r   c                   [        U R                  S   5      nU R                  nU R                  R                  nU[
        R                  :X  a  [        U R                  5      nO[        S5      (       d  [        S5      e[        R                  " 5       n[
        R                  " XE5      n[        R                  " XpR                  R                  R                   XCUR                  R                   [        R"                  5        [$        R&                  R(                  R*                  R-                  XX R                  [/        [1        U SS5      5      S9nU$ )a(  Converts a CSR-matrix to COO format.

Args:
    x (cupyx.scipy.sparse.csr_matrix): A matrix to be converted.
    data (cupy.ndarray): A data array for converted data.
    indices (cupy.ndarray): An index array for converted data.

Returns:
    cupyx.scipy.sparse.coo_matrix: A converted matrix.

r   r   zcsr2coo is not available.rv   Fr0  )ra   r`   ri   rg   rD   rP   rQ   rj   r   r   r   r   r   r   xcsr2coor}   r   rg  r  r  r  r  r|   boolr~   )	rK   r}   rO   r   ri   r  rk  r   As	            r   r   r     s     	AGGAJA
%%CIEKKQXX&!),,:;;,,.kk#)HHMM%%ssxx||..	0 	%%117GG!A.68 	2 	9A Hr   c           	     n   U R                   nU R                  R                  nUS:X  ai  UR                  [        R
                  " SU R                  5      [        R
                  " SU5      [        R                  " US-   U5      U R                  SS9$ [        U R                  X$5      n[        U R                  5      n[        R                  " [        R                  " X`R                  /5      5      n[        U SS5      nUR                  U R                  U   Xg   XPR                  U(       a  SSS9$ SSS9$ )a  Pure-CuPy CSR<->CSC transpose for int64 indices.

Uses ``_build_indptr`` + ``lexsort`` -- O(nnz log nnz) time.
Used as the int64 fallback because no Generic API CSR<->CSC
function exists as of CUDA 13.2 / dev.  This is the biggest
perf gap: int64 tocsc is 7-10x slower than int32 (measured on
Blackwell, 10K-100K matrices).

Args:
    x: Input compressed sparse matrix.
    output_cls: Output class (csc_matrix or csr_matrix).
    out_dim: Size of the output's major axis (n for CSC, m for CSR).
r   rX   T)rz   rv   Nrx   )ri   rO   rD   r|   rP   r   rn   r`   rt   rj   rg   rb  rc  r~   r}   )	rK   
output_clsout_dimri   r  
out_indptrexpandedr\  	canonicals	            r    _cupy_transpose_compressed_int64r  ?  s     %%C		I
ax%%KK177#KK9%KK!Y/GG# & % 	% qyy'=Jahh'H MM%++x&;<=E 2D9I!!	ux
GG%.T " ! !48 " ! !r   c                `   U R                   R                  [        R                  :X  aE  [	        U [
        R                  R                  R                  [        U R                  S   5      5      $ [        S5      (       d  [        S5      e[        R                  " 5       nU R                  u  p#U R                  n[        R                   " X@R                  5      n[        R                   " US5      nUS:X  a  [        R"                  " US-   S5      nO[        R                   " US-   S5      n[%        SU R                  XX4U R&                  R&                  R(                  U R*                  R&                  R(                  U R                   R&                  R(                  UR&                  R(                  UR&                  R(                  UR&                  R(                  [,        R.                  [,        R0                  5        [
        R                  R                  R                  XVU4U R                  S9$ )NrX   r   zcsr2csc is not available.r  r   r  )rO   rD   rP   rQ   r  r  r  r  r  ra   r`   r   r   r   r   ri   r   rn   r   r}   r   rg   r   CUSPARSE_ACTION_NUMERICrg  rK   r   r   r   ri   r}   rO   rg   s           r   r   r   o  s~   yy%++% 0u{{!!,,c!''!*o? 	? i((677((*F77DA
%%C;;sGG$Dkk#s#G
axQUC(QUC(qwwqqvv{{HHMMqyy~~11IIMM7<<++V[[__--..	0 ;;((	qww ) 0 0r   c                   U R                   R                  [        R                  :X  aE  [	        U [
        R                  R                  R                  [        U R                  S   5      5      $ [        S5      (       d  [        S5      e[        R                  " 5       nU R                  u  p#U R                  n[        R                   " X@R                  5      n[        R                   " US5      nUS:X  a  [        R"                  " US-   S5      nGO[        R                   " US-   S5      n[$        R&                  " U R                  5      n[(        R*                  n	[(        R,                  n
[(        R.                  n[(        R0                  " XX4U R2                  R2                  R4                  U R6                  R2                  R4                  U R                   R2                  R4                  UR2                  R4                  UR2                  R4                  UR2                  R4                  XX5      n[        R                   " U[8        R:                  5      n[(        R<                  " XX4U R2                  R2                  R4                  U R6                  R2                  R4                  U R                   R2                  R4                  UR2                  R4                  UR2                  R4                  UR2                  R4                  XXUR2                  R4                  5        [
        R                  R                  R                  XVU4U R                  S9$ )NrX   r   zcsr2cscEx2 is not available.r  r   r  )rO   rD   rP   rQ   r  r  r  r  r  ra   r`   r   r   r   r   ri   r   rn   r   r   r   r  rg  CUSPARSE_CSR2CSC_ALG1csr2cscEx2_bufferSizer}   r   rg   rG   r*  r   rK   r   r   r   ri   r}   rO   rg   x_dtypeactionibasealgorl  buffers                 r   r   r     *   yy%++%/u{{!!,,c!''!*o? 	? l++9::((*F77DA
%%C;;sGG$Dkk#s#G
axQUC(QUC(&&qww/2222..55qqvv{{0A0AIINN		v{{LLgu< [&++6qqvv{{0A0AIINN		v{{LLguFKKOO	M ;;((	qww ) 0 0r   c                `   [        U R                  S   5      nU R                  nU R                  R                  nU[
        R                  :X  a  [        U R                  5      nO[        R                  " 5       n[
        R                  " XE5      n[        R                  " XpR                  R                  R                  XCUR                  R                  [        R                  5        [         R"                  R$                  R&                  R)                  XX`R                  5      nSUl        U$ )a(  Converts a CSC-matrix to COO format.

Args:
    x (cupyx.scipy.sparse.csc_matrix): A matrix to be converted.
    data (cupy.ndarray): A data array for converted data.
    indices (cupy.ndarray): An index array for converted data.

Returns:
    cupyx.scipy.sparse.coo_matrix: A converted matrix.

rX   F)ra   r`   ri   rg   rD   rP   rQ   rj   r   r   r   r   r  r}   r   rg  r  r  r  r  r|   ry   )	rK   r}   rO   r   ri   r  ru  r   r  s	            r   csc2coor    s     	AGGAJA
%%CIEKKQXX&,,.kk#)HHMM%%ssxx||..	0 	%%11sGG	%A"AHr   c                `   U R                   R                  [        R                  :X  aE  [	        U [
        R                  R                  R                  [        U R                  S   5      5      $ [        S5      (       d  [        S5      e[        R                  " 5       nU R                  u  p#U R                  n[        R                   " X@R                  5      n[        R                   " US5      nUS:X  a  [        R"                  " US-   S5      nO[        R                   " US-   S5      n[%        SU R                  XX$U R&                  R&                  R(                  U R*                  R&                  R(                  U R                   R&                  R(                  UR&                  R(                  UR&                  R(                  UR&                  R(                  [,        R.                  [,        R0                  5        [
        R                  R                  R                  XVU4U R                  S9$ )Nr   r   zcsc2csr is not available.r  rX   r   r  )rO   rD   rP   rQ   r  r  r  r  r  ra   r`   r   r   r   r   ri   r   rn   r   r}   r   rg   r   r  rg  r  s           r   r   r     s|   yy%++%/u{{!!,,c!''!*o? 	? i((677((*F77DA
%%C;;sGG$Dkk#s#G
axQUC(QUC(qwwqqvv{{HHMMqyy~~11IIMM7<<++V[[__--..	0 ;;((	qww ) 0 0r   c                   U R                   R                  [        R                  :X  aE  [	        U [
        R                  R                  R                  [        U R                  S   5      5      $ [        S5      (       d  [        S5      e[        R                  " 5       nU R                  u  p#U R                  n[        R                   " X@R                  5      n[        R                   " US5      nUS:X  a  [        R"                  " US-   S5      nGO[        R                   " US-   S5      n[$        R&                  " U R                  5      n[(        R*                  n	[(        R,                  n
[(        R.                  n[(        R0                  " XX$U R2                  R2                  R4                  U R6                  R2                  R4                  U R                   R2                  R4                  UR2                  R4                  UR2                  R4                  UR2                  R4                  XX5      n[        R                   " U[8        R:                  5      n[(        R<                  " XX$U R2                  R2                  R4                  U R6                  R2                  R4                  U R                   R2                  R4                  UR2                  R4                  UR2                  R4                  UR2                  R4                  XXUR2                  R4                  5        [
        R                  R                  R                  XVU4U R                  S9$ )Nr   r   zcsc2csrEx2 is not available.r  rX   r  )rO   rD   rP   rQ   r  r  r  r  r  ra   r`   r   r   r   r   ri   r   rn   r   r   r   r  rg  r  r  r}   r   rg   rG   r*  r   r  s                 r   r   r     r  r   c                   [        S5      (       d  [        S5      eU R                  S:w  a  [        S5      e[        R
                  " U 5      n [        R                  " SSS9n[        R                  " 5       nU R                  u  p4[        R                  5       n[        R                  " US5      n[        SU R                  U[        R                   X4UR"                  U R$                  R&                  X6R$                  R&                  UR(                  R$                  5        [+        U5      n[        R                  " XR                  5      n[        R                  " US	-   S5      n[        R                  " US5      n	[        SU R                  X#XER"                  U R$                  R&                  X6R$                  R&                  UR$                  R&                  U	R$                  R&                  UR$                  R&                  5        [,        R.                  R0                  R3                  XyU4U R                  S
9n
SU
l        U
$ )zConverts a dense matrix in CSC format.

Args:
    x (cupy.ndarray): A matrix to be converted.

Returns:
    cupyx.scipy.sparse.csc_matrix: A converted matrix.

r   zdense2csc is not available.r   expected 2-D matrixr   r  r[   ri   rX   r  T)r   r   r   rR   rP   asfortranarrayrG   r   r   r   r`   r   r   r   rD   r   CUSPARSE_DIRECTION_COLUMNr   r}   r   r   ra   r  r  r  r  rv   )rK   ri   r   r   r   r   nnz_per_colr}   rg   rO   cscs              r   r   r     s    k**899vv{.//QA
,,r
%C((*F77DA  "E++a%Kqww	33Q5;K;K	

A''++SZZ__>
 c(C;;sGG$D[[Q$Fkk#s#GQWW1&&	

A''++		w||''	: ++


'
'(?qww
'
OC $CJr   c                   [        S5      (       d  [        S5      eU R                  S:w  a  [        S5      e[        R
                  " U 5      n [        R                  " SSS9n[        R                  " 5       nU R                  u  p4[        R                  5       n[        R                  " US5      n[        SU R                  U[        R                   X4UR"                  U R$                  R&                  X6R$                  R&                  UR(                  R$                  5        [+        U5      n[,        R.                  (       a  US	:X  a  [        S
5      e[        R                  " XR                  5      n[        R                  " US-   S5      n[        R                  " US5      n	[        SU R                  X#XER"                  U R$                  R&                  X6R$                  R&                  UR$                  R&                  UR$                  R&                  U	R$                  R&                  5        [0        R2                  R4                  R7                  XyU4U R                  S9n
SU
l        U
$ )zConverts a dense matrix in CSR format.

Args:
    x (cupy.ndarray): A matrix to be converted.

Returns:
    cupyx.scipy.sparse.csr_matrix: A converted matrix.

r   zdense2csr is not available.r   r  r   r  r[   ri   r   @hipSPARSE currently cannot handle sparse matrices with null ptrsrX   r  T)r   r   r   rR   rP   r  rG   r   r   r   r`   r   r   r   rD   r   CUSPARSE_DIRECTION_ROWr   r}   r   r   ra   r   r   r  r  r  r  rv   )rK   ri   r   r   r   r   nnz_per_rowr}   rg   rO   csrs              r   r   r   ;  s    k**899vv{.//QA
,,r
%C((*F77DA  "E++a%Kqww	00!8H8H	

A''++SZZ__>
 c(C!8 > ? ?;;sGG$D[[Q$Fkk#s#GQWW1&&	

A''++		v{{(8(8	: ++


'
'(?qww
'
OC $CJr   c                   [        S5      (       d  [        S5      eU R                  R                  S;  a  [	        S5      e[
        R                  " 5       nU R                  u  p4[        R                  " US5      n[        SU R                  X#U R                  R                  U R                  R                  R                  U R                  R                  R                  UR                  R                  U5	      n[        R                   " X`R                  5      n[        R                  " US-   S5      n[        R                   " US5      n	[        SU R                  X#X@R                  R                  U R                  R                  R                  U R"                  R                  R                  U R                  R                  R                  U R$                  UR                  R                  UR                  R                  U	R                  R                  UR                  R                  U5        [&        R(                  R*                  R-                  XyU4U R                  S9$ )	Nr   z"csr2csr_compress is not available.rY  rZ  r  nnz_compressrX   r  )r   r   rD   r^  rR   r   r   r`   rP   r   r   r   r   r}   r   rg   rn   rO   ri   r  r  r  r  )
rK   tolr   r   r   r  ri   r}   rg   rO   s
             r   r   r   j  s   011?@@ww||6!,--((*F77DA++a%K
188&&	**K,<,<,@,@#GC ;;sGG$D[[Q$Fkk#s#GAGG1hh))	++QXX]]->->	{##TYY]]GLL4D4D ;;((	qww ) 0 0r   c                    U S:X  a  [         R                  $ U S:X  a  [         R                  $ U S:X  a  [         R                  $ [        e)Nuint16int32rQ   )r   CUSPARSE_INDEX_16UCUSPARSE_INDEX_32ICUSPARSE_INDEX_64Ir   r[   s    r   _dtype_to_IndexTyper    sC    +++	'	+++	'	+++r   c                  D    \ rS rSrSS jr\R                  4S jrS rSr	g)BaseDescriptori  Nc                (    Xl         X l        X0l        g r   )r5  getdestroy)r   r   r  	destroyers       r   r   BaseDescriptor.__init__  s    	 r   c                    U" 5       (       a  g U R                   c  S U l        g U R                  b#  U R                  U R                  5        S U l        g g r   )r  r5  r$   s     r   r&   BaseDescriptor.__del__  sE    <<DIYY"LL#DI #r   c                r    U R                   b%  [        U R                  U R                  5      U5      $ [        er   )r  r~   r5  AttributeError)r   r   s     r   __getattr__BaseDescriptor.__getattr__  s,    88488DII.55r   )r5  r  r  )NN)
r=   r>   r?   r@   r   r
   r%   r&   r  rB   r   r   r   r  r    s    !
 (-'='= r   r  c                  *    \ rS rSr\S 5       rS rSrg)r1  i  c                B   [         R                  R                  R                  U5      (       d  [	        S5      eUR
                  u  p#[        R                  n[        R                  " UR                  5      nUR                  S:X  a  [        R                  " X#UR                  UR                  R                  R                   UR"                  R                  R                   UR                  R                  R                   [%        UR                  R                  5      [%        UR"                  R                  5      XE5
      n[        R&                  nGOUR                  S:X  a  [        R(                  " X#UR                  UR*                  R                  R                   UR,                  R                  R                   UR                  R                  R                   [%        UR*                  R                  5      XE5	      n[        R.                  nOUR                  S:X  a  [        R0                  " X#UR                  UR                  R                  R                   UR"                  R                  R                   UR                  R                  R                   [%        UR                  R                  5      [%        UR"                  R                  5      XE5
      nS nO$[3        SR                  UR                  5      5      e[        R4                  n[7        XgU5      $ )Nzexpected sparse matrixr  r&  r  z3csr, csc and coo format are supported (actual: {}).)r  r  r  issparser   r`   r   rg  r   r   rD   rS   	createCsrri   rg   r}   r   rO   r  csrGet	createCoork  ru  cooGet	createCscrR   destroySpMatr1  )	r   rT   r$  r%  idx_baser@  r5  r  r  s	            r   r   SpMatDescriptor.create  s   {{!!**1--455WW
55))!''2
88u&&AEE188==#4#4aiinn6H6H!4QXX^^!D#AIIOO4hLD ""CXX&&AEE155::>>155::>>!4QUU[[!A&D ""CXX&&AEE188==#4#4aiinn6H6H!4QXX^^!D#AIIOO4hLD C --3VAHH-=? ?((t'22r   c                F    [         R                  " U R                  X5        g r   )r   spMatSetAttributer5  )r   	attributer}   s      r   set_attributeSpMatDescriptor.set_attribute  s    ##DIIy?r   r   N)r=   r>   r?   r@   rA   r   r  rB   r   r   r   r1  r1    s    3 3<@r   r1  c                  $    \ rS rSr\S 5       rSrg)rh  i  c           	     z   UR                   n[        R                  " UR                  5      n[        R
                  " X3UR                  R                  UR                  R                  [        UR                  5      [        R                  U5      n[        R                  n[        R                  n[        XVU5      $ r   )sizer   r   rD   r   createSpVecr}   r   r  rg  spVecGetdestroySpVecrh  )r   idxrK   ri   r@  r5  r  r  s           r   r   SpVecDescriptor.create  s    ff))!''2
$$Ssxx||QVVZZ%8%C%.%G%G%/1   ((t'22r   r   Nr=   r>   r?   r@   rA   r   rB   r   r   r   rh  rh    s    	3 	3r   rh  c                  $    \ rS rSr\S 5       rSrg)ri  i  c                   [         R                  " UR                  5      n[        R                  " UR
                  UR                  R                  U5      n[        R                  n[        R                  n[        X4U5      $ r   )r   r   rD   r   createDnVecr  r}   r   dnVecGetdestroyDnVecri  )r   rK   r@  r5  r  r  s         r   r   DnVecDescriptor.create  sY    ))!''2
$$QVVQVVZZD  ((t'22r   r   Nr  r   r   r   ri  ri    s    3 3r   ri  c                  $    \ rS rSr\S 5       rSrg)DnMatDescriptori  c                   UR                   S:w  a  [        S5      eUR                  R                  (       d  [        S5      eUR                  u  p#Un[
        R                  " UR                  5      n[        R                  " X#XAR                  R                  U[        R                  5      n[        R                  n[        R                  n[        XgU5      $ )Nr   r  zexpected F-contiguous array)r   rR   r   r   r`   r   r   rD   r   createDnMatr}   r   CUSPARSE_ORDER_COLdnMatGetdestroyDnMatr  )	r   rT   r$  r%  ldr@  r5  r  r  s	            r   r   DnMatDescriptor.create  s    66Q;233ww##:;;WW
))!''2
$$TVVZZ%.%A%AC  ((t'22r   r   Nr  r   r   r   r  r    s    3 3r   r  c                   [        S5      (       d  [        S5      e[        U [        R                  R
                  R                  5      (       am  U R                  n[        U[        R                  R
                  R                  5      (       d%  SR                  [        U5      5      n[        U5      eUn U(       + n[        U [        R                  R
                  R                  [        R                  R
                  R                  45      (       d#  [        SR                  [        U 5      5      5      eU(       d  U R                  OU R                  SSS2   nUS   [        U5      :w  a  [        S5      eU R                   (       d  [        S	5      eUu  p[#        XU5      u  pnUc!  [$        R&                  " XR(                  5      nO[        U5      U	:w  a  [        S5      eU R*                  S
:X  a  UR-                  S
5        U$ [.        R1                  U 5      n[2        R1                  U5      n[2        R1                  U5      n[4        R6                  " 5       n[9        U5      n[:        R<                  " X0R(                  5      R>                  n[:        R<                  " X@R(                  5      R>                  n[@        RB                  " U R(                  5      n[D        RF                  n[D        RH                  " XURJ                  URL                  URL                  URJ                  URL                  UU5	      n[$        RN                  " U[$        RP                  5      n[D        RR                  " XURJ                  URL                  URL                  URJ                  URL                  UUURJ                  RT                  5
        U$ )a  Multiplication of sparse matrix and dense vector.

.. math::

    y = \alpha * op(A) x + \beta * y

Args:
    a (cupyx.scipy.sparse.csr_matrix, csc_matrix or coo_matrix):
        Sparse matrix A
    x (cupy.ndarray): Dense vector x
    y (cupy.ndarray or None): Dense vector y
    alpha (scalar): Coefficient
    beta (scalar): Coefficient
    transa (bool): If ``True``, op(A) = transpose of A.

Returns:
    cupy.ndarray
r   zspmv is not available."aT must be csr_matrix (actual: {})r  NrY   rX   r   zexpected canonical formatr   )+r   r   r   r  r  r  r  Tr  rS   r  r   r  r`   r   rR   ry   rM   rP   rn   rD   ri   fillr1  r   ri  r   r   r   rG   r   r   r   r   r   CUSPARSE_MV_ALG_DEFAULTspMV_bufferSizer}   r5  r   r*  spMVr   )rT   rK   r   r   r   r   aTr   r   r   r   desc_aro  rp  r   r  r@  rA  r-  r.  s                       r   r   r     s   & f%%344!U[[''2233SS"ekk00;;<<6==d2hGCC. a%++,,77++,,779 : :7>>tAwGHH#agg2GqzSV-..!!455DAa(GA!yKK77#	Q1-..uuz	q	##A&F##A&F##A&F((*F6"DLL(//E<<gg&--D%%agg.J

+
+C))&

*0++v{{DII*0++z3HI ;;y%**-DNN6V[[&++99fkk:sDIIMMK Hr   i c                2
   [        S5      (       d  [        S5      eU R                  S:w  d  UR                  S:w  a  [        S5      eUR                  R
                  (       d  [        S5      eUb&  UR                  R
                  (       d  [        S5      e[        U [        R                  R                  R                  5      (       am  U R                  n[        U[        R                  R                  R                  5      (       d%  SR                  [        U5      5      n[        U5      eUn U(       + n[        U [        R                  R                  R                  [        R                  R                  R                   45      (       d#  [        S	R                  [        U 5      5      5      eU(       d  U R"                  OU R"                  SSS
2   n	U(       d  UR"                  OUR"                  SSS
2   n
U	S   U
S   :w  a  [        S5      eU R$                  (       d  [        S5      eU	u  pU
u  p['        XU5      u  pnUc$  [(        R*                  " X4U R,                  S5      nO1UR"                  S   U:w  d  UR"                  S   U:w  a  [        S5      eU R.                  S:X  a  UR1                  S5        U$ U[2        :  a  [5        5       S:  av  [7        SU[2        5       H_  n[9        U[2        -   U5      nU(       a  [(        R:                  " XU2SS24   5      nOUSS2UU24   nUSS2UU24   n[=        U UUX4XVS9  Ma     U$ [>        RA                  U 5      n[B        RA                  U5      n[B        RA                  U5      n[D        RF                  " 5       n[I        U5      n[I        U5      n[J        RL                  " X0R,                  5      RN                  n[J        RL                  " X@R,                  5      RN                  n[P        RR                  " U R,                  5      n[T        RV                  n[T        RX                  " UUUURZ                  UR\                  UR\                  URZ                  UR\                  UU5
      n[(        R^                  " U[(        R`                  5      n[T        Rb                  " UUUURZ                  UR\                  UR\                  URZ                  UR\                  UUURZ                  Rd                  5      nU$ )a  Multiplication of sparse matrix and dense matrix.

.. math::

    C = \alpha * op(A) op(B) + \beta * C

Args:
    a (cupyx.scipy.sparse.csr_matrix, csc_matrix or coo_matrix):
        Sparse matrix A
    b (cupy.ndarray): Dense matrix B
    c (cupy.ndarray or None): Dense matrix C
    alpha (scalar): Coefficient
    beta (scalar): Coefficient
    transa (bool): If ``True``, op(A) = transpose of A.
    transb (bool): If ``True``, op(B) = transpose of B.

Returns:
    cupy.ndarray
r   zspmm is not available.r   r   r   Nr   r  r  rY   rX   r   r   r  r   i1  )r   r   r   r   r  )3r   r   r   rR   r   r   r   r  r  r  r  r  r  rS   r  r   r  r`   ry   rM   rP   rn   rD   ri   r  _SPMM_MAX_DENSE_COLSr   rangeminr  r   r1  r   r  r   r   r   rG   r   r   r   r   r   CUSPARSE_MM_ALG_DEFAULTspMM_bufferSizer}   r5  r   r*  spMMr   )rT   r   r   r   r   r   r  r  r   r   r  r   r   rP  r   col0col1b_chunkc_chunkr  desc_bdesc_cr   r  r  r@  rA  r-  r.  s                                r   r   r   C  s   ( f%%344vv{affk01177<==}QWW11<==!U[[''2233SS"ekk00;;<<6==d2hGCC. a%++,,77++,,779 : :7>>tAwGHH#agg2G#agg2GqzWQZ-..!!:;;DADAa(GA!yKK-	
qAGGAJ!O-..uuz	q	JL5$8!Q 45Dt22A6D..qdA? AtDyL/49oGGwe/ 6 ##A&F##A&F##A&F((*F6"D6"DLL(//E<<gg&--D%%agg.J

+
+C))&$ejj*0++v{{DII*0++z3HI ;;y%**-DvtT5::v{{%{{DIIv{{)3		?I Hr   c                   [        S5      (       d  [        S5      e[        R                  R                  R                  U 5      (       d9  [        R                  R                  R                  U 5      (       d  [        S5      e[        U[        R                  5      (       d  [        S5      eUR                  S;  a  [        S5      eU R                  S   U R                  S   s=:X  a  UR                  S   :X  d  O  [        S	5      eU R                  UR                  :w  a  [        S
5      e[        U S5        USL a  [         R"                  nO0USL a  [         R$                  nO[        SR'                  U5      5      eUSL a  [         R(                  n	O0USL a  [         R*                  n	O[        SR'                  U5      5      eUSL a  Sn
O"USL a  Sn
O[        SR'                  U5      5      eUSL a  [         R,                  nO0USL a  [         R.                  nO[        SR'                  U5      5      eU R                  nUR0                  S:X  a  SnOSUR0                  S:X  a  SnO@UR0                  S:X  a  SnO-UR0                  S:X  a  SnO[        SR'                  U5      5      e[3        [         US-   5      n[3        [         US-   5      n[3        [         US-   5      nUSL d  US:X  a  [         R4                  nOnUSL d  US:X  a  [         R6                  nORUS:X  a2  UR0                  S;   a  [         R6                  nO+[         R8                  nO[        S R'                  U5      5      e[        R                  R                  R                  U 5      (       an  U[         R8                  :X  a  [        S!5      eU R:                  n [        R                  R                  R                  U 5      (       d  [        S"5      eSU-
  nSU-
  nU R                  S   nUR                  S:X  a  SOUR                  S   nUR<                  (       a  [         R4                  nUnO/UR>                  (       a  [         R6                  nUnO[        S#5      e[@        RB                  " 5       n[D        RF                  " X,S$9n[H        RK                  5       nURM                  [         RN                  5        URQ                  [         RR                  5        URU                  U5        URW                  U	5        [         RX                  " 5       nU" UXUUUU RZ                  UR\                  R^                  UR`                  U R^                  R^                  Rb                  U Rd                  R^                  Rb                  U Rf                  R^                  Rb                  UR^                  Rb                  UUU5      n[        Rh                  " U4[D        Rj                  S$9nU" UXUUUU RZ                  UR\                  R^                  UR`                  U R^                  R^                  Rb                  U Rd                  R^                  Rb                  U Rf                  R^                  Rb                  UR^                  Rb                  UUUUR^                  Rb                  5        U" UXUUUU RZ                  UR\                  R^                  UR`                  U R^                  R^                  Rb                  U Rd                  R^                  Rb                  U Rf                  R^                  Rb                  UR^                  Rb                  UUUUR^                  Rb                  5        [l        Rn                  " 5       Rq                  5         [         Rr                  " U5        g%)&a  Solves a sparse triangular linear system op(a) * x = alpha * b.

Args:
    a (cupyx.scipy.sparse.csr_matrix or cupyx.scipy.sparse.csc_matrix):
        Sparse matrix with dimension ``(M, M)``.
    b (cupy.ndarray): Dense vector or matrix with dimension ``(M)`` or
        ``(M, K)``.
    alpha (float or complex): Coefficient.
    lower (bool):
        True: ``a`` is lower triangle matrix.
        False: ``a`` is upper triangle matrix.
    unit_diag (bool):
        True: diagonal part of ``a`` has unit elements.
        False: diagonal part of ``a`` has non-unit elements.
    transa (bool or str): True, False, 'N', 'T' or 'H'.
        'N' or False: op(a) == ``a``.
        'T' or True: op(a) == ``a.T``.
        'H': op(a) == ``a.conj().T``.
    blocking (bool):
        True: blocking algorithm is used.
        False: non-blocking algorithm is used.
    level_info (bool):
        True: solves it with level information.
        False: solves it without level information.

Note: ``b`` will be overwritten.
r   zcsrsm2 is not available.z"a must be CSR or CSC sparse matrixzb must be cupy.ndarray)rX   r   b.ndim must be 1 or 2r   rX   zinvalid shapedtype mismatchTFUnknown lower (actual: {})Unknown unit_diag (actual: {})zUnknown blocking (actual: {})Unknown level_info (actual: {})r   r   r   r   r   r   r   Invalid dtype (actual: {})csrsm2_bufferSizeExtcsrsm2_analysiscsrsm2_solveNr  HfdzUnknown transa (actual: {})z@If matrix is CSC format and complex dtype,transa must not be 'H'z#expected CSR matrix after transpose'b must be F-contiguous or C-contiguous.r[   N):r   r   r  r  r  isspmatrix_csrisspmatrix_cscrR   r   rP   ndarrayr   r`   rD   r   rV   r   CUSPARSE_FILL_MODE_LOWERCUSPARSE_FILL_MODE_UPPERrS   CUSPARSE_DIAG_TYPE_NON_UNITCUSPARSE_DIAG_TYPE_UNITCUSPARSE_SOLVE_POLICY_NO_LEVELCUSPARSE_SOLVE_POLICY_USE_LEVELr^  r~   r   r   &CUSPARSE_OPERATION_CONJUGATE_TRANSPOSEr  _f_contiguous_c_contiguousr   r   rG   r   r   r   r+   CUSPARSE_MATRIX_TYPE_GENERALr0   rg  r5   r;   createCsrsm2Infori   r   r}   r   r   rg   rO   r   r*  _streamget_current_streamsynchronizedestroyCsrsm2Info)rT   r   r   lower	unit_diagr   blocking
level_infor4   r:   r  policyrD   thelperanalysissolver   nrhsr  r   r   a_descrQ  ws_sizewss                             r   r   r     s   : h''566KK--a00KK--a00=>>a''122vvV011GGAJ!''!*2
2))ww!''())H%}66		%66	5<<UCDDE99		d	55	9@@KLL5	T	8??IJJU99	t	:::AA*MNNGGEzzS	s		s		s	4;;EBCCY$: :;Fy!&7"78HIq>12E&C-;;	46S=77	3::;;FEEF6==fEFF{{((++YEEE 8 9 9CC{{!!0033ABBV	M		
A!1D;;	
77BCC((*FLL,E!!#F
	>>?
i@@A
Y'
Y'%%'DVT61dAEE\\&&(9(9166;;??XX]]&&		(:(:AFFJJ6#G 
gZv{{	3BVT61dAEE5<<;L;L1B1BYY^^S$M 
&$48I8I


QVV[[__ahhmm.?.?
))..

affjj#tVRWW[[J
  ,,.%r   c                (	   [        S5      (       d  [        S5      e[        R                  R                  R                  U 5      (       d  [        S5      eU R                  S   U R                  S   :w  a$  [        SR                  U R                  5      5      e[        U S5        USL a  [        R                  nO0USL a  [        R                  nO[        S	R                  U5      5      eU R                  nUR                  S
:X  a  SnOSUR                  S:X  a  SnO@UR                  S:X  a  SnO-UR                  S:X  a  SnO[        SR                  U5      5      e[!        [        US-   5      n[!        [        US-   5      n[!        [        US-   5      n[!        [        S5      n["        R$                  " 5       n	U R                  S   n
U R&                  n[(        R+                  5       nUR-                  [        R.                  5        UR1                  [        R2                  5        [        R4                  " 5       nU" XXR6                  U R8                  R8                  R:                  U R<                  R8                  R:                  U R>                  R8                  R:                  U5      n[@        RB                  " U4[D        RF                  S9n[D        RB                  " S[D        RH                  S9nU" XXR6                  U R8                  R8                  R:                  U R<                  R8                  R:                  U R>                  R8                  R:                  XUR8                  R:                  5
         U" XURJ                  R8                  5        U" XXR6                  U R8                  R8                  R:                  U R<                  R8                  R:                  U R>                  R8                  R:                  XUR8                  R:                  5
         U" XURJ                  R8                  5        g! [L         a    [        SR                  US   5      5      ef = f! [L         a    [        SR                  US   5      5      ef = f)a  Computes incomplete LU decomposition for a sparse square matrix.

Args:
    a (cupyx.scipy.sparse.csr_matrix):
        Sparse matrix with dimension ``(M, M)``.
    level_info (bool):
        True: solves it with level information.
        False: solves it without level information.

Note: ``a`` will be overwritten. This function does not support fill-in
    (only ILU(0) is supported) nor pivoting.
r   zcsrilu02 is not available.za must be CSR sparse matrixr   rX   zinvalid shape (a.shape: {})FTr  r   r   r   r   r   r   r   r  csrilu02_bufferSizecsrilu02_analysisxcsrilu02_zeroPivotr[   rX   za({0},{0}) is missingzu({0},{0}) is zeroN)'r   r   r  r  r  r  r   r`   rR   rS   rV   r   r  r  rD   r^  r~   r   r   ri   r   r   r+   r  r0   rg  createCsrilu02Infor   r}   r   rg   rO   rP   r   rG   r*  r  r   	Exception)rT   r"  r#  rD   r$  r%  r&  r'  checkr   r   ri   r5  rQ  r*  r+  positions                    r   r   r   6  s`    j))788;;,,Q//566wwqzQWWQZ6==aggFGGJ'U99	t	:::AA*MNNGGEzzS	s		s		s	4;;EBCCY$9 9:Fy!&9"9:HIq:~.EI45E((*F	
A
%%C!Di<<=I>>?'')DV__affkkooXX]]&&		(:(:DBG	gZv{{	3B||D5HV__affkkooXX]]		 2 2D"''++OFfHOO001 
&S//166;;??
((--

QYY^^//rww{{LCfHOO001  F077DEEF  C-44Xa[ABBCs   <P>  Q) >(Q&)(Rc                	   [        S5      (       d  [        S5      eU R                  S:w  a  [        S5      eU R                  R
                  S;  a  [        S5      e[        R                  " U 5      n [        R                  U 5      nUS:X  a=  [        R                  R                  R                  U R                  U R                  S9nOUS	:X  a=  [        R                  R                  R                  U R                  U R                  S9nO]US
:X  a=  [        R                  R                  R!                  U R                  U R                  S9nO[#        SR%                  U5      5      e[&        R                  U5      n[(        R*                  n[,        R.                  " 5       n[(        R0                  " XbR2                  UR2                  U5      n[        R4                  " U[        R6                  5      n[(        R8                  " XbR2                  UR2                  XXR:                  R<                  5        [>        R@                  " SSS9n	[>        R@                  " SSS9n
[>        R@                  " SSS9n[(        RB                  " UR2                  U	RD                  R:                  U
RD                  R:                  URD                  R:                  5        [G        U5      n[H        RJ                  (       a  US:X  a  [        S5      eUS:X  ay  URL                  n[        R4                  " US5      n[        R4                  " XR                  5      n[        R                  R                  R                  XU4U R                  S9nGOUS	:X  ax  URL                  n[        R4                  " US5      n[        R4                  " XR                  5      n[        R                  R                  R                  XU4U R                  S9nOUS
:X  a  [        RN                  " US5      n[        RN                  " US5      n[        R4                  " XR                  5      n[        R                  R                  R!                  XU44U R                  S9n[&        R                  U5      n[(        RP                  " XbR2                  UR2                  XXR:                  R<                  5        SUl)        U$ )a  Converts a dense matrix into a CSR, CSC or COO format.

Args:
    x (cupy.ndarray): A matrix to be converted.
    format (str): Format of converted matrix. It must be either 'csr',
        'csc' or 'coo'.

Returns:
    cupyx.scipy.sparse.spmatrix: A converted sparse matrix.

r   zdenseToSparse is not available.r   r  rY  rZ  r  r[   r  r&  zunsupported format (actual: {})r   rQ   r  r  r  T)*r   r   r   rR   rD   r^  rP   r  r  r   r  r  r  r  r`   r  r  r   rS   r1  r   "CUSPARSE_DENSETOSPARSE_ALG_DEFAULTr   r   denseToSparse_bufferSizer5  r   r*  denseToSparse_analysisr}   r   rG   r   r7  r   ra   r   r   rg   rn   denseToSparse_convertry   )rK   rS   ro  r   rp  r  r   r-  r.  num_rows_tmpnum_cols_tmpri   rg   rO   r}   rk  ru  s                    r   r   r   ~  s    o..<==vv{.//ww||6!,--QA##A&FKK))!'')A	5KK))!'')A	5KK))!'')A9@@HII##A&F77D((*F226;;39;;FI;;y%**-D$$V[[%+[[$		G<<1L<<1L
,,q
(C6;;(;(;(@(@'..33SZZ__F
c(C!8 > ? ?++c3'{{3(KK))4&*A01 * 9	5++c3'{{3(KK))4&*A01 * 9	5kk#s#kk#s# {{3(KK))4s*<AGG)L##A&F##FKK$*KKyy}}F
 "AHr   c                t   [        S5      (       d  [        S5      eU R                  nUR                  S;  a  [	        S5      eUc!  [
        R                  " U R                  USS9nOAUR                  R                  (       d  [	        S5      eUR                  U:w  a  [	        S5      e[        R                  U 5      n[        R                  U5      n[        R                  n[        R                   " 5       n[        R"                  " XcR$                  UR$                  U5      n[
        R&                  " U[
        R(                  5      n[*        R,                  (       a  U R.                  S	:X  a  [	        S
5      e[        R0                  " XcR$                  UR$                  XXR2                  R4                  5        U$ )a  Converts sparse matrix to a dense matrix.

Args:
    x (cupyx.scipy.sparse.spmatrix): A sparse matrix to convert.
    out (cupy.ndarray or None): A dense matrix to store the result.
        It must be F-contiguous.

Returns:
    cupy.ndarray: A converted dense matrix.

r   zsparseToDense is not available.rY  rZ  r   r[  r]  zout dtype mismatchr   r  )r   r   rD   r^  rR   rP   rn   r`   r   r   r1  r   r  r   "CUSPARSE_SPARSETODENSE_ALG_DEFAULTr   r   sparseToDense_bufferSizer5  r   r*  r   r   ri   r   r}   r   )	rK   rm   rD   ro  desc_outr  r   r-  r.  s	            r   r   r     sF    o..<==GGEzz,--
{kk!''c:yy%%BCC99122##A&F%%c*H77D((*F226;;3;==$HI;;y%**-D55A: > ? ?FKK$MM4@ Jr   c                   [        S5      (       d  [        S5      eUSL a  SnOUSL a  SnOUS;  a  [        SU S	35      e[        R                  R
                  R                  U 5      (       a  O[        R                  R
                  R                  U 5      (       aT  US:X  a  U R                  n SnO7US:X  a  U R                  n SnO"US
:X  a  U R                  5       R                  n SnU(       + nO:[        R                  R
                  R                  U 5      (       a  O[        S5      eU R                  (       d  [        S5      eUR                  S:X  a  SnUR                  SS5      nOUR                  S:X  a  SnO[        S5      eU R                  S   U R                  S   s=:X  a  UR                  S   :X  d  O  [        S5      eU R                  nUR                   S;  a  [#        SR%                  U5      5      eXqR                  :w  a  [#        S5      eUSL a  [&        R(                  nO0USL a  [&        R*                  nO[        SR%                  U5      5      eUSL a  [&        R,                  n	O0USL a  [&        R.                  n	O[        SR%                  U5      5      eUS:X  a  [&        R0                  n
OHUS:X  a  [&        R2                  n
O1UR                   S;   a  [&        R2                  n
O[&        R4                  n
UR6                  (       a  [&        R0                  nO]UR8                  (       aA  [&        R:                  " 5       S:  a  [        S5      eUR                  n[&        R2                  nO[        S5      eU R                  u  pU[&        R0                  :X  a  UR                  u  pOUR                  u  pX4n[<        R>                  " XR                  SS9n[@        RB                  " 5       n[D        RG                  U 5      n[H        RG                  U5      n[H        RG                  U5      n[&        RJ                  " 5       n[L        RN                  " UUR                  S9RP                  n[R        RT                  " UR                  5      n[&        RV                  n URY                  [&        RZ                  U5        URY                  [&        R\                  U	5        [&        R^                  " UXUR`                  URb                  URb                  URb                  UUU5
      n[<        Rd                  " U[<        Rf                  S9n[&        Rh                  " UXUR`                  URb                  URb                  URb                  UUUUR`                  Rj                  5        [&        Rl                  " UXUR`                  URb                  URb                  URb                  UUUUR`                  Rj                  5        U(       a  UR                  S5      nU[&        Rn                  " U5        $ ! [&        Rn                  " U5        f = f)a  Solves a sparse triangular linear system op(a) * x = alpha * op(b).

Args:
    a (cupyx.scipy.sparse.csr_matrix or cupyx.scipy.sparse.coo_matrix):
        Sparse matrix with dimension ``(M, M)``.
    b (cupy.ndarray): Dense matrix with dimension ``(M, K)``.
    alpha (float or complex): Coefficient.
    lower (bool):
        True: ``a`` is lower triangle matrix.
        False: ``a`` is upper triangle matrix.
    unit_diag (bool):
        True: diagonal part of ``a`` has unit elements.
        False: diagonal part of ``a`` has non-unit elements.
    transa (bool or str): True, False, 'N', 'T' or 'H'.
        'N' or False: op(a) == ``a``.
        'T' or True: op(a) == ``a.T``.
        'H': op(a) == ``a.conj().T``.
r   zspsm is not available.Fr	  Tr  NTHzUnknown transa (actual: )r
  z'a must be CSR, CSC or COO sparse matrixr  rX   rY   r   r   r   rK  rY  r  r  r  r  r  i-  zb must be F-contiguous.r  r   r[  r[   )8r   r   rR   r  r  r  r  r  r  conjisspmatrix_coory   r   reshaper`   rD   r^  r   rS   r   r  r  r  r  r   r   r  r  r  r   rP   rn   r   r   r1  r   r  spSM_createDescrrG   r   r   r   r   CUSPARSE_SPSM_ALG_DEFAULTr  CUSPARSE_SPMAT_FILL_MODECUSPARSE_SPMAT_DIAG_TYPEspSM_bufferSizer}   r5  r   r*  spSM_analysisr   
spSM_solvespSM_destroyDescr)rT   r   r   r  r   r   is_b_vectorrD   r4   r:   r  r  r   rP  r   c_shaper   r   r;  r<  r=  
spsm_descrr@  r  r-  r.  s                             r   r   r     s   & f%%344 	4	u	3F81=>> {{((++				*	*1	-	-S=AFs]AFs]

AF					*	*1	-	-BCC!!:;; 	vv{IIb!	
1011 GGAJ!''!*2
2+,, GGEzz4;;EBCC()) }66		%66	5<<UCDD E99		d	55	9@@KLL }99	355::99DCCD 	99	
&&(50677CC55BCC 77DAy999ww1wwdGG77#6A ((*F""1%E""1%E""1%E++-JLLagg.55E%%agg.J..D0I>>	J 	I>>	J --D

EJJ

JJ
D*6	 {{9EJJ7 	D

EJJ

JJ
D*diimm	E
 	D

EJJ

JJ
D*diimm	E
 		"A 	##J/	##J/s   *E8X9 9Yc                4   [         R                  nU R                  R                  USS9nU R                  R                  USS9nUR                  R                  USS9nUR                  R                  USS9nU R
                  S   nUR
                  S   n	[         R                  " U5      n
X   n[        UR                  5       5      nUS:X  a  [        R                  R                  R                  R                  [         R                  " SU R                  5      [         R                  " SU5      [         R                   " US-   U5      X45      $ [         R"                  " [         R$                  " U R&                  US9U5      n[         R                   " U R&                  S-   US9n[         R(                  " XSS S9  [         R$                  " XS9X   -
  nA[+        U5      nXM   nUU   nUU-   nUU   nUU   nU R,                  U   UR,                  U   -  nAAAAAUS:w  a  UU R                  R/                  U5      -  n[        R                  R                  R0                  R                  UUUX4S9nUR3                  5         UR5                  5       $ )	zPure-CuPy sort-merge SpGEMM for int64: C = alpha * A * B.

Used on CUDA < 13.0 where cuSPARSE spGEMM lacks int64.
Expands all (i,j,a*b) products into COO, then sum_duplicates.
O(P log P) time, O(P + nnz_C) space, where P = total products.
Fr^   r   rX   r[   Nrl   r  )rP   rQ   rO   rI   rg   r`   rf   ra   sumr  r  r  r  r|   r   rD   rn   re   rc   ri   rq   rj   r}   r  r  r  r  )rT   r   r   r  	a_indicesa_indptr	b_indicesb_indptrr   r   	b_row_lenproducts_per_atotal_productsa_srccum_prodb_offsetr"  a_col_kb_row_start	b_col_posc_colsc_rowsc_valsr&  s                           r   _cupy_spgemm_int64rc  	  sH    I		   7Ixxyu5H		   7Ixxyu5H	
A	
A 

8$I)N++-.N{{!!,,88KK177#KK9%KKAy)F	 	 LLQUU),n>E {{15519I6H	LLab\2^5G  H%F G7#Kh&Iy!FE]FVVE]QVVI..FxFGz!'',,u-- ++


'
'
3
3qf 4 .C99;r   c                   U R                   S:w  d  UR                   S:w  a  [        S5      e[        U [        R                  R
                  R                  5      (       d#  [        SR                  [        U 5      5      5      e[        U[        R                  R
                  R                  5      (       d#  [        SR                  [        U5      5      5      eU R                  R                  [        R                  :X  d(  UR                  R                  [        R                  :X  a  U R                  S   UR                  S   :w  a  [        S5      e[        R                   (       + =(       a*    [        R"                  " 5       S:  =(       a    [%        S5      =(       d    [%        S	5      nU(       a5  ['        U [        R                  5      n ['        U[        R                  5      nO[)        X5      u  p[+        XU5      $ [%        S5      (       d  [-        S
5      eU R.                  (       d  [        S5      eUR.                  (       d  [        S5      eU R                  S   UR                  S   :w  a  [        S5      eU R                  u  pEUR                  u  pg[0        R2                  " U R                  R                  UR                  R                  5      n[)        X5      u  pXG4n	[        R4                  " US-   US9n
[        R                  R
                  R                  R7                  [        R8                  " SU R                  5      [        R8                  " SU5      X5      n[:        R<                  " 5       n[>        RA                  U 5      n[>        RA                  U5      n[>        RA                  U5      n[B        RD                  " 5       n[B        RF                  n[B        RF                  n[0        RH                  " X+R                  S9RJ                  n[0        RH                  " SUR                  S9RJ                  n[L        RN                  " UR                  5      n[B        RP                  nSn [B        RR                  " UUUURT                  URV                  URV                  URT                  URV                  UUUSU5      n[        R8                  " U[        RX                  5      n[B        RR                  " UUUURT                  URV                  URV                  URT                  URV                  UUUUURT                  RZ                  5        [B        Rb                  " UUUURT                  URV                  URV                  URT                  URV                  UUUSU5      n[        R8                  " U[        RX                  5      n[B        Rb                  " UUUURT                  URV                  URV                  URT                  URV                  UUUUURT                  RZ                  5        [0        RH                  " SSS9n[0        RH                  " SSS9n[0        RH                  " SSS9n[B        Rd                  " URV                  URJ                  RT                  URJ                  RT                  URJ                  RT                  5        U	S   [g        U5      :X  d   eU	S   [g        U5      :X  d   e[g        U5      nURh                  n[        R8                  " UU5      n [        R8                  " UUR                  5      n![B        Rj                  " URV                  URT                  RZ                  U RT                  RZ                  U!RT                  RZ                  5        [B        Rl                  " UUUURT                  URV                  URV                  URT                  URV                  UUU5        [        R                  R
                  R                  R7                  U!U UU	SSS9n[B        Rn                  " U5        U$ ! [B        R\                   Ga  n[_        5       S:  a  Ue[B        R`                  n[B        RR                  " UUUURT                  URV                  URV                  URT                  URV                  UUUSU5      n[        R8                  " U[        RX                  5      n[B        RR                  " UUUURT                  URV                  URV                  URT                  URV                  UUUUURT                  RZ                  5         SnAGNSnAff = f)a  Matrix-matrix product for CSR-matrix.

math::
   C = alpha * A * B

Args:
    a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A.
    b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B.
    alpha (scalar): Coefficient

Returns:
    cupyx.scipy.sparse.csr_matrix

r   r   r  rX   r   rK  r   r   r   zspgemm is not available.r  r	  r[   r   NrQ   Trx   )8r   rR   r   r  r  r  r  r   rS   r  rO   rD   rP   rQ   r`   r   r   runtimeGetVersionr   r   rM   rc  r   ry   rG   r  rn   r|   r   r   r   r1  r   r   spGEMM_createDescrr   r   r   r   r   CUSPARSE_SPGEMM_DEFAULTspGEMM_workEstimationr}   r5  r*  r   CuSparseErrorr   CUSPARSE_SPGEMM_ALG2spGEMM_computer7  ra   rg   r8  spGEMM_copyspGEMM_destroyDescr)"rT   r   r   _native_int64r   r   rP  r   r  rO  c_empty_indptrr   r   r;  r<  r=  spgemm_descrr  r  r   r@  r  r,  
buff1_sizebuff1cse
buff2_sizebuff2rD  rE  rG  r  r  r  s"                                     r   r   r   	  s    	vv{affk011a++66777>>tAwGHHa++66777>>tAwGHH 	yy%++%EKK)G771:#/00 -**,5-"8,0  /	 	
  $Au{{3A#Au{{3A$Q*DA%aE22h''566!!:;;!!:;;wwqzQWWQZ+,,77DA77DA""199??AIIOODIQ"DAfG[[Qi8N 	%%11AqwwQ	!:	!A ((*F""1%E""1%E""1%E//1L55D55DLLgg.55E<<)00D%%agg.J,,DH44D$

EJJ

DIIJJ
D,8E
 J

3''D$

EJJ

DIIJJ
D,
JJNN	* ))dEJJ

EJJ		

JlAxAJ KK
EJJ/EdEJJ

EJJ		

JlJ

P
 aw/Jaw/JLL'*E5::z'8'8'='=%,,115<<3D3DF1:Z(((1:Z(((JExxHE9-I[[(FUZZ):):INN<N<N#[[__. dEJJ

EJJ		

Jl4 	%%11	8W!d 	2 	<A !!,/Ha ""  <%I--44D$

EJJ

DIIJJ
D,8E
 J

3''D$

EJJ

DIIJJ
D,
JJNN	 	s   7C!a e."De))e.r   )returnra   )NrX   r   F)NrX   r   T)N      ?g        FF)rX   rX   )FF)NrX   rX   )rw  )NrX   r   FF)rw  TFFTF)F)r  )rw  TFFr0  )W
__future__r   	functoolsrE   numpyrG   platformr   cupyrP   cupy_backends.cuda.apir   r   r   r   cupy_backends.cuda.libsr   r   
cupy._corer   	cupy.cudar   r   r	   r  r
   cupyx.scipy.sparser  r   rM   rV   rj   rt   r   r   r   r   infr   r   memoizer   r   r   r   r   r   r   r   r'  r   r   r   r   r   r   r   r   r   r   r  r   r  r   r   r  r   r   r   r   r   r  r  r1  rh  ri  r  r   r  r   r   r   r   r   r   rc  r   r   r   r   <module>r     s=   "     4 6 9  ' '  = =B'>B	-*:5]5}5 ]5 m	5
 }5 5 }5 5 M5 u/65 u/65  !5" #5$ |%5& |'5( |)5* |+5, |-5. }/50 }152 -354 -556 758 95: ;5< m=5> ?5@ ]A5B ]C5D mE5F MG5T mU5h Mi5 p%  [%  

D!	% 
 [%  k%  {%  %  {%  %  K%  K%  K%  %  %   {!% " {#% $ {%% & {'% ( {)% * {+% , {-% . 6::t$/% 0 6::t$1% 2 3% 4 5% 6 7% 8 k9% : ;% < [=% > [?% @ vzz4 A% B C% F vzz4 G% H VZZ&I%  P  (?0f>BJ8v@FAH6Nbl^GTnbDD+;\+;\@'F(&(&$N-!`0> 0F>0: 0F(V,^0: ,"@n "@J3n 33n 33n 3$AN " \~ AF%*Q&hECPJZ(V`0F;|Nr   