+jcXddlZddlZddlZddlZddlZddlmZddlZddl m Z ddl m Z m Z mZddlmZddlZddlZddlZddlZddlZddlmZmZddlmZmZddlZddlZddl Z ddl!m"Z"m#Z#ddl$m%Z%dd l&m'Z'm(Z(m)Z)dd lm*Z*Gd d ej+j,Z-d8d Z.d9dZ/Gddej0j1Z2Gddejj3Z4dZ5 d:dejj3fdZ6 d;dej3de7de8ej3fdZ9Gddej:j;Zd Z?d!Z@d"ZAd#ZBd$ZCd%ZDd&ZEd'ZFd(ZGd)ZHd*ZId+ZJd,ZKGd-d.e(jLZMGd/d0e(jLZNGd1d2e(jLZOGd3d4e(jLZPGd5d6e'ZQdeQfd7ZRdS)=N)trange)Image ImageDraw ImageFont)override) CONDRegularCONDList)argsPerformanceFeature)adapters adapter_maps)BypassInjectionManager)ComfyExtensionioui) ProgressBarc:eZdZdZddfd Z ddZxZS) TrainGuiderzB CFGGuider with modifications for training specific logic F offloadingcHtj|i|||_dSN)super__init__r)selfrr kwargs __class__s 8/home/wildlama/comfy/ComfyUI/comfy_extras/nodes_train.pyrzTrainGuider.__init__"s*$)&)))$Nc tj|j|j|j|j|j |j\|_|_|_ tj |jj } |&tj||j| }|| }|| }|| }tj|j| |j |j|||| |||||||  } |jn#|jwxYwtj|j|j |`|` | S)N)force_full_load force_offload)devicedtype) latent_shapes)comfysampler_helpersprepare_sampling model_patchershapeconds model_optionsr inner_model loaded_modelstorchcuda empty_cache load_device prepare_masktosamplerscast_to_load_options model_dtypepre_run inner_samplecleanupcleanup_models) rnoise latent_imagesamplersigmas denoise_maskcallback disable_pbarseedr%r#outputs r outer_samplezTrainGuider.outer_sample&s  ! 2 2"  "$(O 3"o 3   9$*d&8     #/  # 0==ek6L  #v.. 6"" ++  vT5G5S5S5U5U ,    )   & & ( ( (&&+'  F   & & ( ( ( (D  & & ( ( ( ( ,,TZ9KLLL     s (8E::F)NNFNN)__name__ __module__ __qualname____doc__rrE __classcell__rs@rrrsv*/%%%%%%% 99999999rrci}|D]\}}ttrt||}n{ttjr$|d|kr|}n=tttfr!t|krfd|D}|||<|S)N full_sizerc g|] }| SrP).0ivs r z0make_batch_extra_option_dict..ls+++QAaD+++r) items isinstancedictmake_batch_extra_option_dictr/Tensorsizelisttuplelen)dindiciesrNnew_dictknewvrSs @rrXrXbsH  1 a   ,/8yQQQDD 5< ( ( , AFF1II$:$:{ D%= ) ) ,c!ff .A.A++++(+++D Orc \t|drt |j tjr|j |_ t |ttfr-t|D]\}}t||d|d|-|S)N__iter__rU.cng|]2}t|tjr|n|3SrP)rVr/rYdetachrQts rrTz%process_cond_list..}s5[[[q 1el(C(CJ!((***[[[r)hasattr enumerateprocess_cond_listr[rUrVrWr/rYcloner condrrhr\)r^prefixindexitemrarSs rrmrmqsq*Ega&9&9E$Q<< 9 9KE4 dv$7$7$7$7 8 8 8 8 G   EOO E EDAq!T"" E!!____5555Au|,, Ewwyy!Ax(( E[[TUTZ[[[A{++ Eafel33-V]]__AFAe}-- E#,Q<<EEKE4%dv,C,C,C,CE,C,CDDDD Hrc heZdZdddddejdddf dZdZ ddZd Zd Z d Z d Z dd Z dS) TrainSamplerNrFc <||_||_||_||_||_||_||_||_| |_| |_ | rtj nd|_ | || dSd|_d|_d|_dSr)loss_fn optimizer loss_callback batch_size total_stepsgrad_accrCtraining_dtype real_datasetbucket_latentsr/amp GradScaler grad_scaler_init_bucket_databucket_offsetsbucket_weights num_images) rrwrxryrzr|r{rCr}r~ruse_grad_scalers rrzTrainSampler.__init__s "*$&   ,7C  6EN59//111$  %  " "> 2 2 2 2 2"&D "&D "DOOOrc8dg|_g}|D]U}||jd|j|jd|jdzV|jd|_t j|tj|_dS)z3Initialize bucket offsets and weights for sampling.rr$N)rappendr*rr/tensorfloat32r)rr bucket_sizeslats rrzTrainSampler._init_bucket_datas c ! O OC    ! - - -   & &t':2'>1'M N N N N-b1#l.s'B'B'BAQ'B'B'BrpositiverMrT)r-model_sampling noise_scalingr/ zeros_liker+rXautocastr#typer}requires_grad_rwfloatr|rscalebackward)r model_wrap batch_sigmas batch_noise batch_latentror_ extra_args dataset_sizebwdxtx0batch_extra_argsx0_predlossbwd_losss ` rfwd_bwdzTrainSampler.fwd_bwds # 2 @ @ +|U   # 2 @ @  \ * *  [ ) )     (C'B'B'B'B'B'B $7 L   ^BIN$2E F F F = = j!!$''++D11#G << <.sZ     " 1 B B 4  %%''     r)ranger/rr4)rrrzr#rs ` r_generate_batch_sigmasz#TrainSampler._generate_batch_sigmassU    :&&    |L)),,V444rc tj|jd}|j|}|jd} |j|t|j| } tj | d|  } fd| D} ||  |} | d| i | j }||| | j }||||| || ||jd }|jr'||||d|d dS) z)Execute one training step in bucket mode.rurNcg|]}|zSrPrP)rQidx bucket_offsets rrTz8TrainSampler._train_step_bucket_mode..sLLLCMC/LLLrsamplesTr.4f)rbucket)r/ multinomialrrrrr*rminrzrandpermtolistr4generate_noiser#rrrry set_postfix)rrrornoisegenr=pbar bucket_idx bucket_latent bucket_sizeactual_batch_sizerelative_indicesabsolute_indicesrrrrrs @r_train_step_bucket_modez$TrainSampler._train_step_bucket_modes&t':A>>CCEE +J7 #)!, +J7  == >+667I8I7IJQQSSLLLL;KLLL$%5699,GG --y,.GHHKK    22:?PR^Reff ||        O      ,   tyy{{ + + + TYY[["6"6*MMNNNNNrc Xtj|d|j}tjfd|D} |d| i| j} ||t|j|| j} | || | | ||||d } |j r'| | | d| didS)zGExecute one training step in standard (non-bucket, non-multi-res) mode.Nc g|] }| SrPrP)rQrRr=s rrTz:TrainSampler._train_step_standard_mode..s#F#F#FLO#F#F#FrrTrrr)r/rrzrstackrr4r#rrrryrrr) rrrorrr=rrr_rrrrs ` r_train_step_standard_modez&TrainSampler._train_step_standard_modes0>,//0A$/0ABIIKK{#F#F#F#FX#F#F#FGG --y,.GHHKK    22:s4?T`?a?acocvww ||              ,   tyy{{ + + + &TYY[["6"6788888rc tj|d|j}d} |D]} |j| |} |d| i| j} |jj tj d } tj | g| j} ||| | | || g||d }| |z } | |jz t!|z } |j-|j| n| |jr'|| |d| didS) zIExecute one training step in multi-resolution mode (real_dataset is set).NrrrFrrr)r/rrzrr~r4rr#r-rrrrrrrr|r]rrrryr)rrrorrr=rrr_ total_lossrq single_latentrrrs r_train_step_multires_modez&TrainSampler._train_step_multires_mode's>,//0A$/0ABIIKK   E -e477 EEM"11M*b%&& &5FFJt$$))++  !<77::=;OPPL<<  D $ JJ$-/#h--?   '   " ": . . 7 7 9 9 9 9    ! ! !   2   z00 1 1 1 &Z__%6%6"<"<=>>>>>rc t|j|_|jd} |d} tjt |j} t|jddtj j x} D]} tj |j| dzz}|j||| |||| n>|j||| |||| | n||| |||| | | dz|jzdkr|j|j|j|jjD]H}|dD]=}|j |jj|jj|j_>I|j9|j|j|jn|j|j | dtjtj!|S) Nrrz Training LoRAg{Gz?)desc smoothingdisableiruparams)"rmr+rZr/r0r1rr{rr&utilsPROGRESS_BAR_ENABLED comfy_extrasnodes_custom_samplerNoise_RandomNoiserCrrr~rrr|runscale_rx param_groupsgraddatar4r$stepupdate zero_gradr)rrr?rrAr<r=r@rBrorui_pbarrrRrrparams rsamplezTrainSampler.sampleKsp-Z-=>>  +{{1~~     d.// $!K<<  D! ! A$8JJ AH$H".,,Zz8Uacghhhh"*..z4XWceqswxxxx..z4XWceqswxxxA&!++#/$--dn===$(N$?OOL!-h!7OO :-$*/*/*<*&>>>rcV|||S)N)r#)r4r)rr#s rmove_tozBiasDiff.move_tos( v((***r)rFrGrHrrrrrJrKs@rrrse;;;???+++++++rrcF d\}}tjd||fd}tj|}t |t |c fd|D}d|t|d|zz f}t|dddD]R\}} t||dz z |z} |t| |zz } | || | fgdd | | f}S|S) N)ii,RGBwhitec&g|] }|z z z SrPrPrQlmax_lossmin_losss rrTz#draw_loss_graph..s'UUUaALX%89UUUrrrustartbluefillwidth) rnewrDrawrvaluesmaxintrlline)loss_mapstepsrheightimgdraw scaled_loss prev_pointrRrxyrrs @@rdraw_loss_graphrs0ME6 )EE6?G 4 4C >#  DX__..//X__5F5F1G1GHhUUUUU8??CTCTUUUKVc+a.6"9:::;J+abb/3331 UQY%' ( ( SV__ $ :1v&V1 ===V Jrmodelc|g}nt|dr|t|tjjtjjtjjfs<||tj d|d|j j d|S|pd}| D]\}}t|||d||S)NforwardzFound module with forward:  ()rootrf)rkrVr/nn ModuleList Sequential ModuleDictrloggingdebugrrFnamed_children*find_all_highest_child_module_with_forward)rresultname next_namechilds rr!r!s~  " ": #UX%8%(:MN,,  e WDWWEO6D!0022YY 525&TBWBWIBWBWXXXX Mrrudepthreturnc |g}|pd}t|tjtjtjf}t |do| }|rJ|dz }||kr?||tjd|d|d|j j d|S| D]\}}t|||||d ||S) a Find modules at a specific depth level for gradient checkpointing. Args: model: The model to search depth: Target depth level (1 = top-level blocks, 2 = their children, etc.) result: Accumulator for results current_depth: Current recursion depth name: Current module name for logging Returns: List of modules at the target depth NrrruzFound module at depth z: rrrf) rVrrrrrkrrrrrFr find_modules_at_depth) rr&r" current_depthr# is_container has_forwardr$r%s rr)r)s ~ >6DebmR]BM%RSSL%++@L0@K E ! ! MM% M_5__D__EOD\___ ` ` `M"0022ZZ 5eUFMdCXCXYCXCXYYYY Mrc^eZdZdZedejfdZedejfdZdS)OffloadCheckpointFunctiona Gradient checkpointing that works with weight offloading. Forward: no_grad -> compute -> weights can be freed Backward: enable_grad -> recompute -> backward -> weights can be freed For single input, single output modules (Linear, Conv*). rc||||_tj5||cdddS#1swxYwYdSr)save_for_backward forward_fnr/no_grad)ctxrr1s rrz!OffloadCheckpointFunction.forwards a   # ]__ ! !:a== ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s AA A grad_outc0|j\}|j}d|_tj5|d}||}|||j}dddn #1swxYwY~~~|dfS)NT) saved_tensorsr1r/ enable_gradrhrrr)r3r4rr1 x_detachedrgrad_xs rrz"OffloadCheckpointFunction.backwards  ^     % %22488J :&&A JJx _F  % % % % % % % % % % % % % % % z:t|sABB  B N) rFrGrHrI staticmethodr/rYrrrPrrr.r.sp! !!!\!  \rr.Fct|dsdS|j|rBt|tjtjtjtjfrfd}n fdfd}|_||_dS)Nrc:t|Sr)r.apply)r org_forwards rcheckpointing_fwdz patch..checkpointing_fwds,221kBB Brc|i|SrrP)r rr>s rfwdzpatch..fwd s;/// /rcTtjj||dS)NF) use_reentrant)r/r checkpoint)r rrAs rr?z patch..checkpointing_fwds&;)44S$V[4\\ \r) rkrrVrLinearConv1dConv2dConv3dr>)mrr?rAr>s @@rpatchrJs 1i )K ]jRY 29bi$PQQ ] C C C C C C 0 0 0 0 0 ] ] ] ] ] AM!AIIIrcFt|dr|j|_|`dSdS)Nr>)rkr>r)rIs runpatchrLs0q-  M MMMrcJg}|D]}||d|S)zProcess latents for bucket mode training. Args: latents: list[{"samples": tensor}] where each tensor is (Bi, C, Hi, Wi) Returns: list of latent tensors rr)latentsr latent_dicts r_process_latents_bucket_moderQs:N66 k)45555 rct|dkr|ddSg}|D]S}|d}|jd}|dkr!|D]}||d>||T|S)zProcess latents for standard (non-bucket) mode training. Args: latents: list of latent dicts or single latent dict Returns: Processed latents (tensor or list of tensors) rurrN)r]r*r)rO latent_listlatentbs sub_latents r_process_latents_standard_moderW*s 7||qqz)$$K'' " \!_ 77$ 5 5 "":d#34444 5   v & & & & rct|dkr|dSg}|D]B}t|tr||-||C|S)zProcess conditioning - either single list or list of lists. Args: positive: list of conditioning Returns: Flattened conditioning list rur)r]rVr[extendr)r flat_positiveros r_process_conditioningr[Bs{ 8}}{M'' dD ! ! '   & & & &   & & & & rcB|rfd|D}t|}td|D}d}tjd|d|dt |D]$\}}tjd|d|j%|||fSt |trt}fd |D}|D]} | | jtjd |t|d krd }nd}tj |d }t|}ngt |tj r%| }|jd }d}n(tjdt|d }d}|||fS)aConvert latents to dtype and compute image counts. Args: latents: Latents (tensor, list of tensors, or bucket list) dtype: Target dtype bucket_mode: Whether bucket mode is enabled Returns: tuple: (processed_latents, num_images, multi_res) c:g|]}|SrPr4rQrjr$s rrTz._prepare_latents_and_count..e#0001144;;000rc30K|]}|jdVdS)rN)r*ris r z-_prepare_latents_and_count..gs(55555555rFz Bucket mode: z buckets, z total samplesz Bucket z: shape c:g|]}|SrPr^r_s rrTz._prepare_latents_and_count..rr`rzLatent shapes: ruTr)dimzInvalid latents type: )r]sumrrrlr*rVr[setaddr/catrYr4errorr) rOr$ bucket_mode num_bucketsr multi_resrRr all_shapesrTs ` r_prepare_latents_and_countrnXs .0000000'll 55W55555   WkWWZWWWXXX(( > >FAs MtG}}>>???  J ))rc|r|Stjd|dt|t|dkr |dkr||zSt||kr#tdt|d|d|S)a[Validate conditioning count matches image count, expand if needed. Args: positive: Conditioning list num_images: Number of images bucket_mode: Whether bucket mode is enabled Returns: Validated/expanded conditioning list Raises: ValueError: If conditioning count doesn't match image count zTotal Images: z, Total Captions: ruzNumber of positive conditions (z#) does not match number of images (z).)rrr] ValueError)rrrjs r!_validate_and_expand_conditioningrqs MP:PPXPPQQQ 8}}j1nn*$$ X* $ $ nc(mm n n`j n n n    Orc|dkridfStjd|}t|dddd}i}|rtj|}||fS)zLoad existing LoRA weights if provided. Args: existing_lora: LoRA filename or "[None]" Returns: tuple: (existing_weights dict, existing_steps int) [None]rloras_steps_rr) folder_pathsget_full_path_or_raisersplitr&rload_torch_file) existing_lora lora_pathexisting_stepsexisting_weightss r_load_existing_lorar~s  1u 3G]KKI,,Y77:@@EEbIJJNB ;66yAA ^ ++rc|d}|jj}i}tjd|d|t |dkrt ||dd} ||dd} d} tD]} | ||| | } | n| t|} | (| |} n0| |j|d  |} | D]\}}|||d |<| d |fSt jt!j|jj|d }t)|d }|||d <||fS)aCreate a weight adapter for a module with weight. Args: module: The module to create adapter for module_name: Name of the module existing_weights: Dict of existing LoRA weights algorithm: Algorithm name for new adapters lora_dtype: dtype for LoRA weights rank: Rank for new LoRA adapters Returns: tuple: (train_adapter, lora_params dict) .weightzCreating weight adapter for z with shape rz.alpha?z .dora_scaleN)rankalpharfTr$ requires_gradz.diff)weightr*rrr]rgetr loadr to_trainr4 create_trainnamed_parameterstrainrr/r Parameterzerosr)module module_namer} algorithm lora_dtyperkeyr* lora_paramsr dora_scaleexisting_adapter adapter_cls train_adapterr# parameterdiff diff_modules r_create_weight_adapterrs   ! ! !C M EK MIII%IIJJJ 5zzQ&**c>>>3??@@%))S*=*=*=tDD  #  K*//-uj    +,  #&y1K  ',5577:::FFMM(44 D5bnn  -==?? = =OD)3|dkr!tj||S|dkr!tj||S|dkr!tj||S|dkr!tj||SdS)zCreate optimizer based on name. Args: optimizer_name: Name of optimizer ("Adam", "AdamW", "SGD", "RMSprop") parameters: Parameters to optimize learning_rate: Learning rate Returns: Optimizer instance Adam)lrAdamWSGDRMSpropN)r/optimrrrr)optimizer_name parameters learning_rates r_create_optimizerrcs{ }=== 7 " "{    >>> 5 {zm<<< 9 $ ${"":-"@@@ % $rc&|dkrtjS|dkrtjS|dkrtjS|dkrtjSdS)zCreate loss function based on name. Args: loss_function_name: Name of loss function ("MSE", "L1", "Huber", "SmoothL1") Returns: Loss function instance MSEL1HuberSmoothL1N)r/rMSELossL1Loss HuberLoss SmoothL1Loss)loss_function_names r_create_loss_functionrxsU""x!!! t # #x   w & &x!!### z ) )x$$&&& * )rcVtjt|}tj|}|r\|ddd|ddd} ||d| i| |||j dS|rT|d|ddd}||d|i||||j dS||d|i||||j dS)aGExecute the training loop. Args: guider: The guider object train_sampler: The training sampler latents: Latent tensors num_images: Number of images seed: Random seed bucket_mode: Whether bucket mode is enabled multi_res: Whether multi-resolution mode is enabled rNrur)rC) r/rrrrrrepeatrrrC) guider train_samplerrOrrCrjrlr?r< dummy_latents r_run_training_looprsb\% ++ , ,F  - ? ? E EE qz"1"~,,ZAqAA   )\!: ; ;           !*##J1a88  )W!5 6 6            )W!5 6 6         rc:eZdZedZedZdS) TrainLoraNodec ftjdddddtjddtjdd tjd d tjd d d ddtjdd d ddtjddd ddtjddddddtjddd d d!tjd"gd#d$d%&tjd'gd(d)d*&tjd+d,d,d-d.tjd/gd0d1d2&tjd3d1d4gd1d5&tj d6d7d89tjd:ttj ttj d,d;&tj dd d d?d@tj dAd7dB9tjdCtjdDdEgzdEdF&tj dGd7dH9tj dId7dJ9gtjdKdLdMNtjdOdPdQNtjddRNgSS)TNrz Train LoRAmodel/trainingTrzThe model to train the LoRA on.tooltiprOzEThe Latents to use for training, serve as dataset/input of the model.rz.The positive conditioning to use for training.rzrui'z#The batch size to use for training.defaultrrrgrad_accumulation_stepsiz>The number of gradient accumulation steps to use for training.r iz*The number of steps to train the LoRA for.rgMb@?gHz>rz&The learning rate to use for training.)rrrrrrzThe rank of the LoRA layers.rx)rrrrrz"The optimizer to use for training.)optionsrr loss_function)rrrrrz&The loss function to use for training.rCrlzbThe seed to use for training (used in generator for LoRA weight initialization and noise sampling)r})bf16fp32nonerzThe dtype to use for training. 'none' preserves the model's native compute dtype instead of overriding it. For fp16 models, GradScaler is automatically enabled.rrzThe dtype to use for lora.quantized_backwardFztWhen using training_dtype 'none' and training on quantized model, doing backward with quantized matmul when enabled.rrrz"The algorithm to use for training.gradient_checkpointingz(Use gradient checkpointing for training.checkpoint_depthz'Depth level for gradient checkpointing.rz@Offload model weights to CPU during training to save GPU memory.rzrtrsz9The existing LoRA to append to. Set to None for new LoRA.rjzeEnable resolution bucket mode. When enabled, expects pre-bucketed latents from ResolutionBucket node. bypass_modezEnable bypass mode for training. When enabled, adapters are applied via forward hooks instead of weight modification. Useful for quantized models where weights cannot be directly modified. LORA_MODELloraz LoRA weights display_namerLOSS_MAPr z Loss historyzTotal training steps)node_idrcategoryis_experimental is_input_listinputsoutputs)rSchemaModelInputLatent ConditioningIntFloatComboBooleanr[r keysrvget_filename_listCustomOutputclss r define_schemazTrainLoraNode.define_schemasy#%% w0QRR c %%(X&  A  -\  H #"!"D  : ???#@  #>>>!D   *A $444"   #V,"8     (!S!  !2!4!455 !2!4!455a8@     , F!  &E     !^! #(:7CCxjP$W     !!D!    !![!gxt ,''..!'/ *%%,,!+^- 7d|dS)NrrN)rr s rryz,TrainLoraNode.execute..loss_callbacks" ''-----r)ryrzr|r{rCr}rr)ryrzr|r{rCr}r~rz[BypassMode] Injected z bypass hooksz![BypassMode] Ejected bypass hooks)8r&model_managementtraining_fp8_bwdrQrWr[r/inference_modern node_helpersstring_to_torch_dtypeset_model_compute_dtyper get_dtypefloat16rr Fp16Accumulationr fastrwarningrnrqrrr~rrrrrrr)diffusion_modelinfor]rJr0r1load_models_gpurtr set_condscreate_injectionsinjectget_hook_count in_trainingrejectmodulesrLr4rhr NodeOutput)/rrrOrrzr rrrrxrrCr}rrrrrrrzrjrrrrrr$r7 latents_dtyperrlr}r|rrr criterionmodules_to_patchrIryrrbypass_injections injectionrrr s/ @rexecutezTrainLoraNode.executeGsG4a] a"9!"<%a( Aw"1*1-Aw'*] /2aL !7!:] +A.%a( !!n !!n 2D/  >27;;GG4W==G)22  !% ( (] L] LD11B#O%;JGGJ''$:>JJ**51111!h0022 %-//!ME"U^33*.):diGGL"NE&+'%9%9EEu~M-G .. *GZ 9:{[[H H # #E * * 0 0 2 2 20C=/Q/Q , n"N  >????Z()Z@@<,nn0D()Z00,, * 0 0-I..@AAI& 4#8H,4D$$$  |EUAVAV||jz||}}})44A! 33333 J " " $ $ $  " 2 2d N 3    J " " $ $ $|H . . . . .  ,"/)4 %(? ?#(#*$3 ! ! ! !-"/)4 %(? ?#(,5!?4$3 ! ! ! ! ;;;F   X & & &!% )$2$D$DRX$N$N!!2))I$$R(((( e~7T7T7V7Veeefff 59&2"!6;&2$0%6,, !++++M"EFFF))++AAJJJJ 6;&2$0%6,, !++++M"EFFF))++AAJJJJy  H H!(!2!2:!>!>!E!E!G!G.  &&u---G#=(EN4JKK{] L] L] L] L] L] L] L] L] L] L] L] L] L] L] L] L] L] Ls./MW&S+A-WA.UA.WWWNrFrGrH classmethodrrrPrrrrsVI I [I VULUL[ULULULrrc<eZdZedZeddZdS)LoraModelLoaderctjddddtjddtjdd d tjd d d ddtjdddgtjddgS)Nr#zLoad LoRA Modelz model/loadersTrz0The diffusion model the LoRA will be applied to.rrrz/The LoRA model to apply to the diffusion model.strength_modelrgYgY@zGHow strongly to modify the diffusion model. This value can be negative.rbypassFzWhen enabled, applies LoRA in bypass mode without modifying base model weights. Useful for training and when model weights are offloaded.rzThe modified diffusion model.r)rrrrrr)rrrrrrrrrs rrzLoraModelLoader.define_schema!sy%*$ %W ,''--$U.$e    !h!*!(2Q 3    rFc|dkrtj|S|r'tj|d||d\}}n&tj|d||d\}}tj|S)Nr)rrr&sdload_bypass_lora_for_modelsload_lora_for_models)rrrr%r& model_lorars rrzLoraModelLoader.executeCs Q  ='' '  !H@@tT>1MJ"H99tT>1MJ}Z(((rNFr rPrrr#r# sN  [ B ) ) )[ ) ) )rr#c<eZdZedZeddZdS)SaveLoRActjddgddddtjddd tjd d d tjdddggS)Nr.z export lorazSave LoRA Weightszadvanced/model_mergingTrrz>The LoRA model to save. Do not use the model with LoRA layers.rrpzloras/ComfyUI_trained_loraz*The prefix to use for the saved LoRA file.rr zYOptional: The number of steps the LoRA has been trained for, used to name the saved file.)optionalr)rsearch_aliasesrrris_output_noderr)rrrrStringrrs rrzSaveLoRA.define_schemaTsy)?,-  ,''--\. 8H   !w /    rNc4tj}tj||\}}}}} | |d|dd} n |d|d|dd} tj|| } t j|| tj S)Nr05z _.safetensorsru) rvget_output_directoryget_save_image_pathospathjoin safetensorsr/ save_filerr) rrrpr  output_dirfull_output_folderfilenamecounter subfolderfilename_prefixoutput_checkpoints rrzSaveLoRA.executeps!688  ,VZ @ @ JHgy/ =#+ G Gg G G G G  #+ U Ue U UG U U U U GLL);=NOO##D*;<<<}rrr rPrrr.r.SsM  [ 6   [   rr.c<eZdZedZeddZdS) LossGraphNodectjdgdddddtjddd tjd d d ggtjjtjjg S)NrE)ztraining chartztraining visualizationz plot losszPlot Loss GraphrTrrzLoss map from training node.rrB loss_graphz&Prefix for the saved loss graph image.r) rr1rrrr2rrhidden)rrrrr3Hiddenprompt extra_pnginfors rrzLossGraphNode.define_schemasy#TTT*%  *%%++$B, %(D  I$bi&=>%    rNc|d}d\}}d}tjd||z||zfd} tj| } t |t |cfd|D} t |} ||t| d|zz f} t| ddd D]R\}}|t|| z |zz}|t||zz }| | ||fgd d ||f} S| |df||fgd d | ||f||z|fgd d d} tj dd}n##t$rtj }YnwxYw| d|d zfd|d | |d z|dzfd|d | |dz dfd|d | |dz |dz fd|d tj| tjdz }t'j|d}t+jt/j||S)Nr)i i(rrc&g|] }|z z z SrPrPrs rrTz)LossGraphNode.execute..s'SSS!H H)<=SSSrrrurrrrblackz arial.ttf rLoss)fontr Stepsz.2fgo@rr)r)rrrrrrr]rrlr rtruetypeIOError load_defaulttextnparrayastyperr/ from_numpyrrr PreviewImage)rrrBrJrK loss_valuesrr marginr rrr rrRrrrrR img_array img_tensorrrs @@rrzLossGraphNode.executes6l  vi EFNFVO4g  ~c"" --s;/?/?(SSSSS{SSS K  fs;q>F+B'C'CCD k!""oQ777  DAqQY.///AQZ(A IIzAq6*qI A A AQJJ FA; 01q III f 7 8wa     ,%k266DD , , ,)++DDD , 1fk"FG DDD 5A:v{+W4g NNN 6B;"x$5$5Dw OOO b[&2+ &8(9(97    HSMM((44u< %i007 }  D D DEEEEs E""FFNNr rPrrrErEsS  [ ,0F0F0F[0F0F0FrrEcLeZdZedeeejfdZdS)TrainingExtensionr'c:KttttgSr)rr#r.rErs r get_node_listzTrainingExtension.get_node_lists       rN) rFrGrHrr[rr ComfyNodergrPrrreresE  T$r|*<%=   X   rrec"KtSr)rerPrrcomfy_entrypointrjs   rr)rcrc)ruNrNr,)Srr8numpyrZr;r/torch.nnrtorch.utils.checkpoint tqdm.autorPILrrrtyping_extensionsrcomfy.samplersr&comfy.sampler_helperscomfy.sd comfy.utilscomfy.model_management comfy.condsrr comfy.cli_argsr r !comfy_extras.nodes_custom_samplerrrvrcomfy.weight_adapterr r comfy.weight_adapter.bypassrcomfy_api.latestrrrrr Guider_BasicrrXrmr5SamplerrtModulerrr!rr[r)autogradFunctionr.rJrLrQrWr[rnrqr~rrrrrrrrhrr#r.rErerjrPrrrs  ++++++++++&&&&&&--------33333333((((77777777>>>>>>3333333333######BBBBB,3@BBBJ        ,w.w.w.w.w.5>)w.w.w.t+++++ux+++"&/3 8?$JN## 9# # ")_####L""""" 7"""J""""*   0,-*-*-*`6,,,*9(9(9(x$$$&"("("(J484848nAAA*'''&, , , ^cLcLcLcLcLBLcLcLcLL 0)0)0)0)0)bl0)0)0)f)))))r|)))XIFIFIFIFIFBLIFIFIF^         1r