3jZSSKrSSKrSSKrSSKrSSKrSSKJr SSKrSSK J r SSK J r J r Jr SSKJr SSKrSSKrSSKrSSKrSSKrSSKJrJr SSKJrJr SSKrSSKrSSK r SSK!J"r"J#r# SSK$J%r% SS K&J'r'J(r(J)r) SS KJ*r* "S S \RVRX5r-S5S jr.S6Sjr/"SS\R`Rb5r2"SS\RRf5r4Sr5S7S\RRf4Sjjr6S8S\RfS\7S\8\Rf4Sjjr9"SS\RtRv5rSr?Sr@SrAS rBS!rCS"rDS#rES$rFS%rGS&rHS'rIS(rJS)rK"S*S+\(R5rM"S,S-\(R5rN"S.S/\(R5rO"S0S1\(R5rP"S2S3\'5rQS\Q4S4jrRg):N)trange)Image ImageDraw ImageFont)override) CONDRegularCONDList)argsPerformanceFeature)adapters adapter_maps)BypassInjectionManager)ComfyExtensionioui) ProgressBarcH^\rSrSrSrSS.U4SjjrSSjrSrU=r$) TrainGuiderz: CFGGuider with modifications for training specific logic F offloadingc2>[TU]"U0UD6 XlgN)super__init__r)selfrr kwargs __class__s 8/home/wildlama/comfy/ComfyUI/comfy_extras/nodes_train.pyrTrainGuider.__init__"s $)&)$c [RRURURUR UR UR(+URS9uUlUlUl [RR5 URRn Ub*[RRXQRU 5nURU 5nURU 5nURU 5n[R R#UR XRR%5S9 URR'5 UR)UUU UUUUUUU S9 n URR+5 [RR-UR UR5 U?U? U $!URR+5 f=f)N)force_full_load force_offload)devicedtype) latent_shapes)comfysampler_helpersprepare_sampling model_patchershapeconds model_optionsr inner_model loaded_modelstorchcuda empty_cache load_device prepare_masktosamplerscast_to_load_options model_dtypepre_run inner_samplecleanupcleanup_models) rnoise latent_imagesamplersigmas denoise_maskcallback disable_pbarseedr'r%outputs r outer_sampleTrainGuider.outer_sample&s  ! ! 2 2""  ""$(OO 3"oo 3  9$*d&8   ##//  # 00==kk6L #v. 6" ++   v5G5G5S5S5U ,  )    & & (&&+' F    & & ( ,,TZZ9K9KL        & & (s 2GG.)r-r/r0r)NNFNN) __name__ __module__ __qualname____firstlineno____doc__rrG__static_attributes__ __classcell__rs@rrrs2*/%% 99r!rc0nUR5HupEUn[U[5(a [XQUS9nOz[U[R 5(aUbUR S5U:XaXQnO>[U[[45(a#[U5U:XaUVs/sHouUPM nnXcU'M U$s snf)N full_sizer) items isinstancedictmake_batch_extra_option_dictr1Tensorsizelisttuplelen)dindiciesrSnew_dictkvnewvis rrWrWbsH  a  /yQD 5<< ( ( AFF1I$:{ D%= ) )c!f .A"*+(QaD(D+  O,s$B>c [US5(a7[US5(d&[U5Hup#[X1SU35 M U$[US5(Ga[UR 55GHpupE[ U[ 5(a[XQSU35 M-[ U[R5(aUR5X'M`[ U[5(aSURVs/sH4n[ U[R5(aUR5OUPM6 snUl M[ U[5(aN[ UR[R5(a"URR5Ul GM(GM+[ U[[45(dGMI[U5Hup#[X1SUSU35 M GMs U$s snf)N__iter__rT.)hasattr enumerateprocess_cond_listrZrTrUrVr1rXcloner conddetachrr[)r]prefixindexitemr`rats rririqshq*ga&9&9$Q 2"&D "&D "DOr!cNS/Ul/nUHYnURURS5 URRURSURS-5 M[ URSUl[R "U[R S9Ulg)z3Initialize bucket offsets and weights for sampling.rr&N)rappendr,rr1tensorfloat32r)rr~ bucket_sizeslats rrTrainSampler._init_bucket_datas c !C    ! -    & &t':':2'>1'M N"--b1#llCCE ++J7 #))!, ++J7  = >>+67I8IJQQS;KL;KCC/;KL$699,G --y,.GHKK    22:R^ReRef ||        OO        tyy{ + TYY[$5*MN+MsE?c P[R"U5SURR5n[R"UV s/sHoU PM sn 5n UR SU 05R U R5n URU[URU5U R5n URUU U U UUUUSS9 n UR(aURU R55 URSU R5S05 gs sn f)zGExecute one training step in standard (non-bucket, non-multi-res) mode.NrTrrr)r1rryrstackrr6r%rrrrxror)rrrkrrr?rrr^rcrrrrs r_train_step_standard_mode&TrainSampler._train_step_standard_modes>>,/0A$//BIIK{{X#FXOX#FG --y,.GHKK    22:s4??T`?acocvcvw ||                tyy{ + &TYY[$578'$GsD#c [R"U5SURR5nSn UHn URU R U5n UR SU 05R U R5n URRR[R"S5R55n [R"U /5R U R5n URUU U U UU /UUSS9 nX- n M XR- [!U5- n UR"b*UR"R%U 5R'5 OU R'5 UR((aUR)U R55 UR+SU R5S05 g) zIExecute one training step in multi-resolution mode (real_dataset is set).NrrrFrrr)r1rryrr}r6rr%r/rrrrorrr{r\rrrrxr)rrrkrrr?rrr^ total_lossrn single_latentrrrs r_train_step_multires_mode&TrainSampler._train_step_multires_mode's>>,/0A$//BIIK E --e477 EM"11M*b%%& &&55FFJJt$))+  !<<7::=;O;OPL<< D  J-. --/#h-?    '    " ": . 7 7 9    !      z0 1 &Z__%6s$;=>r!c [UR5UlURSn URS5n [RR 5 [ UR5n [URSS[RR(+S9=n GHn [RRURU S--5nUR bUR#XX>Xl5 O6UR$cUR'XX>XjU 5 OUR)XX>XjU 5 U S-UR*-S:XGa3UR,b%UR,R/UR05 UR0R2HfnUSHZnUR4cMUR4R6R9UR6R:5UR4lM\ Mh UR,b@UR,R=UR05 UR,R?5 OUR0R=5 UR0RA5 U R?S5 GM [RR 5 [RB"U5$) Nrrz Training LoRAg{Gz?)desc smoothingdisableirtparams)"rir-rYr1r2r3rrzrr(utilsPROGRESS_BAR_ENABLED comfy_extrasnodes_custom_samplerNoise_RandomNoiserEr~rr}rrr{runscale_rw param_groupsgraddatar6r&stepupdate zero_gradr)rrrArrCr>r?rBrDrkrui_pbarrrcrrparams rsampleTrainSampler.sampleKs--Z-=-=>  +{{1~   d../  $!KK<<<  D  $88JJ AH$H"".,,ZzUah""*..zWcswx..zWcswxA&!+##/$$--dnn=$(NN$?$?L!-h!7 ::-$*/**//*<*[TU]5 Xlgr)rrbias)rrrs rrBiasDiff.__init__s  r!cURnURUR5UR-RU5$r)r&r6r)rb org_dtypes r__call__BiasDiff.__call__s1GG TYY$))+// ::r!clURR5URR5-$r)rnelement element_sizers rpassive_memory_usageBiasDiff.passive_memory_usages'yy!!#dii&<&<&>>>r!c@URUS9 UR5$)N)r%)r6r)rr%s rmove_toBiasDiff.move_tos v((**r!)r) rIrJrKrLrrrrrNrOrPs@rrrs;?++r!rcSup#[R"SX#4S5n[R"U5n[ UR 55[ UR 55pvUR 5Vs/sH oU- Xv- - PM n nSU[U SU-5- 4n [U SSSS9H?up[XS- - U-5n U[X-5- n URXU 4/SSS 9 X4n MA U$s snf) N)ii,RGBwhiterrtstartbluefillwidth) rnewrDrawrvaluesmaxintrhline)loss_mapstepsrheightimgdrawmin_lossmax_lossl scaled_loss prev_pointrcxys rdraw_loss_graphrsME ))EE?G 4C >># DX__./X__5F1GhCK??CTUCTaLX%89CTKUVc+a.6"9::;J+ab/3 QY%' ( S_ $ :1v&V1 =V 4 JVs7C4modelcUc/nO[US5(a[U[RR[RR [RR 45(dDURU5 [R"SUSURRS35 U$U=(d SnUR5Hup4[XAUSU35 M U$)NforwardzFound module with forward:  ()rootrf)rgrUr1nn ModuleList Sequential ModuleDictrloggingdebugrrInamed_children*find_all_highest_child_module_with_forward)rresultname next_namechilds rr'r's~  " ": ##UXX%8%8%((:M:MN,,  e 3D6EOO6D!002 25TF!I;BWX3 Mr!depthreturnc Uc/nU=(d Sn[U[R[R[R45n[ US5=(a U(+nU(aQUS- nX1:XaGUR U5 [R"SUSUSURRS35 U$UR5Hupx[XX#USU35 M U$) ad Find modules at a specific depth level for gradient checkpointing. Args: model: The model to search depth: Target depth level (1 = top-level blocks, 2 = their children, etc.) result: Accumulator for results current_depth: Current recursion depth name: Current module name for logging Returns: List of modules at the target depth rrrtzFound module at depth z: rrrf) rUr r!r"r#rgrr$r%rrIr&find_modules_at_depth) rr,r( current_depthr) is_container has_forwardr*r+s rr/r/s ~ >6DebmmR]]BMM%RSL%+@L0@K  ! MM% MM25'D6EOOD\D\C]]^_ `M"002 eFdV1YKCXY3 Mr!cp\rSrSrSr\S\R4Sj5r\S\R4Sj5r Sr g) OffloadCheckpointFunctioniz Gradient checkpointing that works with weight offloading. Forward: no_grad -> compute -> weights can be freed Backward: enable_grad -> recompute -> backward -> weights can be freed For single input, single output modules (Linear, Conv*). rcURU5 X l[R"5 U"U5sSSS5 $!,(df  g=fr)save_for_backward forward_fnr1no_grad)ctxrr7s rr!OffloadCheckpointFunction.forwards/ a # ]]_a=__s ? A grad_outc8URunURnSUl[R"5 UR 5R S5nU"U5nUR U5 URnSSS5 AAAWS4$!,(df  N=f)NT) saved_tensorsr7r1 enable_gradrlrrr)r9r;rr7 x_detachedrgrad_xs rr"OffloadCheckpointFunction.backwards   ^^     2248J:&A JJx __F ! z:t|! s AB  BrN) rIrJrKrLrM staticmethodr1rXrrrNrr!rr4r4sE! !!  r!r4c,^^[US5(dgURmU(aT[U[R[R [R [R45(aU4SjnO U4SjmU4SjnTUlX lg)Nrc0>[RUT5$r)r4apply)r org_forwards rcheckpointing_fwd patch..checkpointing_fwds,221kB Br!c>T"U0UD6$rr)r rrFs rfwdpatch..fwd s// /r!cV>[RRRTXSS9$)NF) use_reentrant)r1r checkpoint)r rrJs rrGrHs&;;))44S$V[4\ \r!) rgrrUr LinearConv1dConv2dConv3drF)mrrGrJrFs @@rpatchrTse 1i ))KjRYY 299bii$PQQ C 0 ] AM!Ir!cN[US5(aURUlU?gg)NrF)rgrFr)rSs runpatchrVs$q-  MM M!r!cD/nUHnURUS5 M U$)zProcess latents for bucket mode training. Args: latents: list[{"samples": tensor}] where each tensor is (Bi, C, Hi, Wi) Returns: list of latent tensors rr)latentsr~ latent_dicts r_process_latents_bucket_moder[s,N k)45 r!c[U5S:XaUSS$/nUHMnUSnURSnUS:waUHnURUS5 M M<URU5 MO U$)zProcess latents for standard (non-bucket) mode training. Args: latents: list of latent dicts or single latent dict Returns: Processed latents (tensor or list of tensors) rtrrN)r\r,r)rY latent_listlatentbs sub_latents r_process_latents_standard_modera*s 7|qqz)$$K " \\!_ 7$ "":d#34%   v & r!c[U5S:XaUS$/nUH<n[U[5(aURU5 M+UR U5 M> U$)zProcess conditioning - either single list or list of lists. Args: positive: list of conditioning Returns: Flattened conditioning list rtr)r\rUrZextendr)r flat_positiverks r_process_conditioningreBs\ 8}{M dD ! !   &   &  r!cU(aUVs/sHo3RU5PM nn[U5n[SU55nSn[R"SUSUS35 [ U5H+upx[R"SUSUR 35 M- XU4$[U[5(a[5n UVs/sHo3RU5PM nnUHn U RU R 5 M [R"SU 35 [U 5S :aS nOSn[R"US S 9n[U5nOh[U[R5(a#URU5nUR S nSnO&[R"S [U535 S nSnXU4$s snfs snf)aConvert latents to dtype and compute image counts. Args: latents: Latents (tensor, list of tensors, or bucket list) dtype: Target dtype bucket_mode: Whether bucket mode is enabled Returns: tuple: (processed_latents, num_images, multi_res) c3># UHoRSv M g7f)rN)r,).0rps r -_prepare_latents_and_count..gs5WWsFz Bucket mode: z buckets, z total samplesz Bucket z: shape zLatent shapes: rtTr)dimzInvalid latents type: )r6r\sumr$r%rhr,rUrZsetaddr1catrXerrorr) rYr& bucket_moderp num_bucketsr multi_resrcr all_shapesr^s r_prepare_latents_and_countruXs(/0144;0'l 5W55   k]*ZLWX(FA MMIaS < =)I--'4  U (/0144;0F NN6<< (  |45 z?Q IIiiQ/G\ GU\\ * ***U#]]1%   .tG}o>?   ))A11s F?GcU(aU$[R"SUS[U535 [U5S:Xa US:aX-$[U5U:wa[S[U5SUS35eU$)a7Validate conditioning count matches image count, expand if needed. Args: positive: Conditioning list num_images: Number of images bucket_mode: Whether bucket mode is enabled Returns: Validated/expanded conditioning list Raises: ValueError: If conditioning count doesn't match image count zTotal Images: z, Total Captions: rtzNumber of positive conditions (z#) does not match number of images (z).)r$r%r\ ValueError)rrrqs r!_validate_and_expand_conditioningrxs MMN:,.@XPQ 8}j1n$$ X* $-c(m_<_`j_kkm n   Or!cUS:Xa0S4$[R"SU5n[URS5SRS5S5n0nU(a[R R U5nX24$)zLoad existing LoRA weights if provided. Args: existing_lora: LoRA filename or "[None]" Returns: tuple: (existing_weights dict, existing_steps int) [None]rloras_steps_rr) folder_pathsget_full_path_or_raiser splitr(rload_torch_file) existing_lora lora_pathexisting_stepsexisting_weightss r_load_existing_lorarsx 1u 33G]KI,,Y7:@@EbIJN ;;66yA  ++r!cpUS3nURRn0n[R"SUSU35 [ U5S:a[ UR US3S55n UR US3S5n Sn [Hn U RXX5n U cM O U c [Un U b U R5RU5n O*W RURUSS 9RU5n U R5HupXUS U3'M U R5RS 5U4$[ R"R%[ R&"URRUS S 95n[)U5R5RS 5nUXS 3'UU4$)apCreate a weight adapter for a module with weight. Args: module: The module to create adapter for module_name: Name of the module existing_weights: Dict of existing LoRA weights algorithm: Algorithm name for new adapters lora_dtype: dtype for LoRA weights rank: Rank for new LoRA adapters Returns: tuple: (train_adapter, lora_params dict) .weightzCreating weight adapter for z with shape rz.alpha?z .dora_scaleN)rankalpharfTr& requires_gradz.diff)weightr,r$r%r\rgetr loadr to_trainr6 create_trainnamed_parameterstrainrr1r  Parameterzerosr)module module_namer algorithm lora_dtyperkeyr, lora_paramsr dora_scaleexisting_adapter adapter_cls train_adapterr) parameterdiff diff_modules r_create_weight_adapterrs M !C MM  EK MM0\%IJ 5zQ&**cU&>3?@%))SE*=tD  #K*//u   + $  #&y1K  ',557:::FM(44 D5bn  -==?OD3<;-q/ 0 @""$33D9;FFxx!! KK ++:T R tn**,;;DA -1 m5)*K''r!c[RR[R"URR USS95n[ U5R5RS5nUS3U0nXE4$)zCreate a bias adapter for a module with bias. Args: module: The module with bias module_name: Name of the module lora_dtype: dtype for LoRA weights Returns: tuple: (bias_module, lora_params dict) Trz.diff_b) r1r rrrr,rrr)rrrr bias_modulers r_create_bias_adapterrsk 88   FKK%%ZtL D4.&&(77=K!]'*D1K  ##r!c 0n/nURR5Hupx[US5(dMURbG[ XXX45upUR U 5 US3n UR X5 URU 5 [US5(dMURcM[XU5upUR U 5 US3n UR X5 URU 5 M XV4$)a0Setup all LoRA adapters on the model. Args: mp: Model patcher existing_weights: Dict of existing LoRA weights algorithm: Algorithm name for new adapters lora_dtype: dtype for LoRA weights rank: Rank for new LoRA adapters Returns: tuple: (lora_sd dict, all_weight_adapters list) weight_functionrr.bias) r named_modulesrgrrradd_weight_wrapperrrr)mprrrrlora_sdall_weight_adaptersnrSadapterrr bias_adapter bias_paramss r_setup_lora_adaptersrsG&&( 1' ( (xx#"8*z#v&7m%%c3#**73q&!!aff&8,@z,R) {+5k%%c8#**<8!)$  ''r!c 0n/n[5nURR5GHFup[U S5(dMU Rb[ XXX45upUR U 5 URU 5 US3n [U [5(a+URX5 [R"SU 35 O)URXSS9 [R"SU 35 [U S5(dMU RcM[XU5upUR U5 US3n URX5 URU 5 [R"S U 35 GMI XVU4$) aGSetup LoRA adapters in bypass mode. In bypass mode: - Weight adapters (lora/lokr/oft) use bypass injection (forward hook) - Bias/norm adapters (BiasDiff) still use weight wrapper (direct modification) This is useful when the base model weights are quantized and cannot be directly modified. Args: mp: Model patcher existing_weights: Dict of existing LoRA weights algorithm: Algorithm name for new adapters lora_dtype: dtype for LoRA weights rank: Rank for new LoRA adapters Returns: tuple: (lora_sd dict, all_weight_adapters list, bypass_manager) rrz:[BypassMode] Added 1D weight adapter (weight wrapper) for r)strengthz/[BypassMode] Added weight adapter (bypass) for rrz5[BypassMode] Added bias adapter (weight wrapper) for )rrrrgrrrrrUrrr$r% add_adapterrr)rrrrrrrbypass_managerrrSrrrrrs r_setup_lora_adapters_bypassr,sZ(G+-N&&( 1' ( (xx#"8*z#v&#**737mgx00))#7MM$^_b^c"de"..sc.JMM$STWSX"YZq&!!aff&8,@z,R) {+5k%%c8#**<8 UVYUZ[\5)8  77r!cUS:Xa[RRXS9$US:Xa[RRXS9$US:Xa[RR XS9$US:Xa[RR XS9$g)zCreate optimizer based on name. Args: optimizer_name: Name of optimizer ("Adam", "AdamW", "SGD", "RMSprop") parameters: Parameters to optimize learning_rate: Learning rate Returns: Optimizer instance Adam)lrAdamWSGDRMSpropN)r1optimrrrr)optimizer_name parameters learning_rates r_create_optimizerrcs{{ == 7 "{{   >> 5 {{z<< 9 ${{"":"@@ %r!c$US:Xa[RR5$US:Xa[RR5$US:Xa[RR 5$US:Xa[RR 5$g)zCreate loss function based on name. Args: loss_function_name: Name of loss function ("MSE", "L1", "Huber", "SmoothL1") Returns: Loss function instance MSEL1HuberSmoothL1N)r1r MSELossL1Loss HuberLoss SmoothL1Loss)loss_function_names r_create_loss_functionrxssU"xx!! t #xx  w &xx!!## z )xx$$&& *r!c[R"[U55n[RR U5nU(aIUSSSR USSS5n URURSU 05U UUURS9 gU(aFUSR USSS5nURURSU05UUUURS9 gURURSU05UUUURS9 g)a#Execute the training loop. Args: guider: The guider object train_sampler: The training sampler latents: Latent tensors num_images: Number of images seed: Random seed bucket_mode: Whether bucket mode is enabled multi_res: Whether multi-resolution mode is enabled rNrtr)rE) r1rrrrrrepeatrrrE) guider train_samplerrYrrErqrsrAr> dummy_latents r_run_training_looprs \\% + ,F  - - ? ? EEqz"1~,,ZAqA   )\!: ;      !*##J1a8  )W!5 6        )W!5 6     r!c4\rSrSr\S5r\S5rSrg) TrainLoraNodeic![R"SSSSS[RRSSS9[RRSS S9[R RS S S9[R RS S S SSS9[R RSS S SSS9[R RSSS SSS9[RRSSSSSSS9[R RSSS S S!S9[RRS"/S#QS$S%S&9[RRS'/S(QS)S*S&9[R RS+S,S,S-S.S9[RRS//S0QS1S2S&9[RRS3S1S4/S1S5S&9[RRS6S7S8S99[RRS:[[R"55[[R"55S,S;S&9[RRSS S S?S@S9[RRSAS7SBS99[RRSC[R"SD5SE/-SESFS&9[RRSGS7SHS99[RRSIS7SJS99/[R"SK5R!SLSMSN9[R"SO5R!SPSQSN9[R R!SSRSN9/SS9$)TNrz Train LoRAmodel/trainingTrzThe model to train the LoRA on.tooltiprYzEThe Latents to use for training, serve as dataset/input of the model.rz.The positive conditioning to use for training.ryrti'z#The batch size to use for training.defaultrr rgrad_accumulation_stepsiz>The number of gradient accumulation steps to use for training.riz*The number of steps to train the LoRA for.rgMb@?gHz>rz&The learning rate to use for training.)rrr rrrzThe rank of the LoRA layers.rw)rrrrrz"The optimizer to use for training.)optionsrr loss_function)rrrrrz&The loss function to use for training.rErlzbThe seed to use for training (used in generator for LoRA weight initialization and noise sampling)r|)bf16fp32nonerzThe dtype to use for training. 'none' preserves the model's native compute dtype instead of overriding it. For fp16 models, GradScaler is automatically enabled.rrzThe dtype to use for lora.quantized_backwardFztWhen using training_dtype 'none' and training on quantized model, doing backward with quantized matmul when enabled.rrrz"The algorithm to use for training.gradient_checkpointingz(Use gradient checkpointing for training.checkpoint_depthz'Depth level for gradient checkpointing.rz@Offload model weights to CPU during training to save GPU memory.rr{rzz9The existing LoRA to append to. Set to None for new LoRA.rqzeEnable resolution bucket mode. When enabled, expects pre-bucketed latents from ResolutionBucket node. bypass_modezEnable bypass mode for training. When enabled, adapters are applied via forward hooks instead of weight modification. Useful for quantized models where weights cannot be directly modified. LORA_MODELloraz LoRA weights display_namerLOSS_MAPr z Loss historyzTotal training steps)node_idrcategoryis_experimental is_input_listinputsoutputs)rSchemaModelInputLatent ConditioningIntFloatComboBooleanrZr keysr}get_filename_listCustomOutputclss r define_schemaTrainLoraNode.define_schemasyy#%% w0QR c %%(X&  A  -\  H #"!"D  : ?#@  #>!D   *A $4"   #V,"8     (!S!  !2!2!45 !2!2!45a8@     , F!  &E     !^! #(::7CxjP$W     !!D!    !![!gxt ,'..!'/ *%,,!+^- 7TSRU5 g)NrrX)rr s rrx,TrainLoraNode.execute..loss_callbacks ''-r!)rxryr{rzrEr|r~r)rxryr{rzrEr|r}rz[BypassMode] Injected z bypass hooksz![BypassMode] Ejected bypass hooks)8r(model_managementtraining_fp8_bwdr[rarer1inference_moderj node_helpersstring_to_torch_dtypeset_model_compute_dtyper get_dtypefloat16rr Fp16Accumulationr fastr$warningrurxrrrr%rrrr rr/diffusion_modelinfor\rTr2r3load_models_gpurrr set_condscreate_injectionsinjectget_hook_count in_trainingrejectmodulesrVr6rlr NodeOutput)/rrrYrryrrrrrwrrEr|rrrrrrrrqrrrrrr&r9 latents_dtyperrsrrrrr criterionmodules_to_patchrSrxrrbypass_injections injectionrrr s/ @rexecuteTrainLoraNode.executeGs4a] a"9!"<%a( Aw"1*1-Aw'*] /2aL !7!:] +A.%a( !!n !!n 2D/ 27;G4W=G)2  ! !% (D1B#O%;;JGJ'$::>J**51!hh002 %--/!MME"U^^3*.)::diiGL"NNE&+'%9Eu~~M-G . *GZ 9:{[H HH # #E * 0 0 20C=/Q , n"N >??Z()@<,n0D()0,, * 0-I..@AIHH,,4D$  @EUAV@WWijzi{|})A! 3* JJ " " $  " " 2 2d N 3  JJ " " $|H . ,"/)4 % ?#(#*$3 ! !-"/)4 % ?#(,54$3 ! ! ;F   X &!% )$2$D$DRXX$N!!2I$$R("3 6~7T7T7V6WWdef 59&&2"!6;&&2$0%6 !+&7MM"EF))+AAJ,y !(!2!2:!>!E!E!G!/&&u-/$==(EN4JK{) (R6;&&2$0%6 !+&7MM"EF))+AAJ,_) (s'5MV=&T#CVA6U==V VrNrIrJrKrL classmethodrr(rNrr!rrrs/I I VULULr!rc8\rSrSr\S5r\SSj5rSrg)LoraModelLoaderi cj[R"SSSS[RRSSS9[R"S5RS S S9[R RS S S SSS9[R RSSSS9/[RRSSS9/S9$)Nr-zLoad LoRA Modelz model/loadersTrz0The diffusion model the LoRA will be applied to.rrrz/The LoRA model to apply to the diffusion model.strength_modelrgYgY@zGHow strongly to modify the diffusion model. This value can be negative.rbypassFzWhen enabled, applies LoRA in bypass mode without modifying base model weights. Useful for training and when model weights are offloaded.rzThe modified diffusion model.r)rrrrrr)rrrrrrrrrs rrLoraModelLoader.define_schema!syy%*$ %W ,'--$U.$e    !h!*!(2Q 3  r!cUS:Xa[R"U5$U(a%[RR USX#S5upVO$[RR USX#S5upV[R"U5$)Nr)rr"r(sdload_bypass_lora_for_modelsload_lora_for_models)rrrr/r0 model_lorars rr(LoraModelLoader.executeCso Q ==' ' !HH@@tT1MJ"HH99tT1MJ}}Z((r!rNFr*rr!rr-r- s)  B ) )r!r-c8\rSrSr\S5r\SSj5rSrg)SaveLoRAiSc[R"SS/SSSS[R"S5RSSS 9[RRS S S S 9[R RSSSS9//S9$)Nr:z export lorazSave LoRA Weightsz model/mergingTrrz>The LoRA model to save. Do not use the model with LoRA layers.rrmzloras/ComfyUI_trained_loraz*The prefix to use for the saved LoRA file.rrzYOptional: The number of steps the LoRA has been trained for, used to name the saved file.)optionalr)rsearch_aliasesrrris_output_noderr)rrrrStringrrs rrSaveLoRA.define_schemaTsyy)?,$  ,'--\. 8H   !w /  r!Nc8[R"5n[R"X$5upVpxn Uc USUSS3n O USUSUSS3n [RR XZ5n [ RRX5 [R"5$)Nr05z _.safetensorsr|) r}get_output_directoryget_save_image_pathospathjoin safetensorsr1 save_filerr") rrrmr output_dirfull_output_folderfilenamecounter subfolderfilename_prefixoutput_checkpoints rr(SaveLoRA.executeps!668  , ,V @ Jg/ =#+*Agb\ G #+*AeWGGB<} U GGLL);O##D<}}r!rrr*rr!rr:r:Ss(  6  r!r:c8\rSrSr\S5r\SSj5rSrg) LossGraphNodeic$[R"S/SQSSSS[R"S5RSSS 9[RRS S S S 9//[R R [R R/S9 $)NrS)ztraining chartztraining visualizationz plot losszPlot Loss GraphrTrrzLoss map from training node.rrO loss_graphz&Prefix for the saved loss graph image.r) rr=rrrr>rrhidden)rrrrr?Hiddenprompt extra_pnginfors rrLossGraphNode.define_schemasyy#T*%  *%++$B, %(D  II$$bii&=&=>%  r!NcbUSnSupgSn[R"SXh-Xx-4S5n [R"U 5n [ U5[ U5pUV s/sH oU - X- - PM nn [ U5nX[USU-5- 4n[USSSS9HCunn U[UU- U-5-nU[X-5- nU RUUU4/S S S 9 UU4nME U RUS4X4/S S S 9 U RX4Xh-U4/S S S 9 Sn[R"S S5nU RSUS -4SUS S9 U RUS -US-4SUS S9 U RUS- S4U SUS S9 U RUS- US- 4U SUS S9 [R "U 5R#[R$5S- n[&R("U5Sn[*R,"[.R0"UUS9S9$s sn f![a [R"5nGNf=f)Nr)i i(rrrrtrrrrblackz arial.ttf rLoss)fontr Stepsz.2fgo@rr)r)rrrrrr r\r rhr rtruetypeIOError load_defaulttextnparrayastyperr1 from_numpyrr"r PreviewImage)rrrOrXrY loss_valuesrrmarginrrrrrrrrrcrrr` img_array img_tensors rr(LossGraphNode.executesm6l  ii ENFO4g ~~c" -s;/?(GRS{!H )<={ SK s;q>F+B'CCD k!"oQ7DAqQY.//AQZ(A IIzAq6*qI AQJ 8 FA; 01q I  7 8wa   ,%%k26D 1fk"FG D 5A:v{+W4g N 6B;"xnDw O b[&2+ &8C.7  HHSM((4u< %%i07 }}  DEEKT& ,))+D ,sHH H.-H.rNNr*rr!rrSrSs*  ,0F0Fr!rScL\rSrSr\S\\\R4Sj5r Sr g)TrainingExtensionir-c6# [[[[/$7fr)rr-r:rSrs r get_node_listTrainingExtension.get_node_lists       srN) rIrJrKrLrrZrr ComfyNodervrNrr!rrtrts)  T$r||*<%=  r!rtc# [5$7fr)rtrr!rcomfy_entrypointrzs  s r)rr)rtNrNr8)Sr$rEnumpyrhrHr1torch.nnr torch.utils.checkpoint tqdm.autorPILrrrtyping_extensionsrcomfy.samplersr(comfy.sampler_helperscomfy.sd comfy.utilscomfy.model_management comfy.condsrr comfy.cli_argsr r !comfy_extras.nodes_custom_samplerrr}rcomfy.weight_adapterr r comfy.weight_adapter.bypassrcomfy_api.latestrrrrr Guider_BasicrrWrir7SamplerrrModulerrr'r rZr/autogradFunctionr4rTrVr[rarerurxrrrrrrrrrxrr-r:rSrtrzrr!rrs  ++&-3(7>33#B,33@@BJ  ,w.5>>))w.t+uxx+"&/3 88??$JN# 99# # "))_#L" 7 7"J"* 0,-*`6,*9(x$&"(J48nA*'&, ^cLBLLcLL 0)bll0)f)r||)XIFBLLIF^   1r!