From 409fd5050fc19afbcd3f056c464d20842f727fd8 Mon Sep 17 00:00:00 2001 From: Juan Treminio Date: Fri, 1 May 2026 18:41:32 -0500 Subject: [PATCH 1/4] Allow using video for ControlNet for target type image --- .../ComfyUIBackend/WorkflowGeneratorSteps.cs | 10 +++++++++ src/wwwroot/css/genpage.css | 2 +- src/wwwroot/js/genpage/gentab/params.js | 21 ++++++++++++++++--- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs b/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs index bff9711b0..4b21de73a 100644 --- a/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs +++ b/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs @@ -1098,6 +1098,16 @@ bool getBestFor(string phrase) { throw new SwarmUserErrorException("Cannot use ControlNet without a model selected."); } + if (imageNodeActual.DataType == WGNodeData.DT_VIDEO && !g.IsVideoModel()) + { + string singleFrame = g.CreateNode("ImageFromBatch", new JObject() + { + ["image"] = imageNodeActual.Path, + ["batch_index"] = 0, + ["length"] = 1 + }); + imageNodeActual = imageNodeActual.WithPath([singleFrame, 0], WGNodeData.DT_IMAGE); + } if (controlModel.ModelClass?.ID?.EndsWith("/control-diffpatch") ?? false) { string modelPatchLoader = g.CreateNode("ModelPatchLoader", new JObject() diff --git a/src/wwwroot/css/genpage.css b/src/wwwroot/css/genpage.css index d3a23ee9b..5313c18f5 100644 --- a/src/wwwroot/css/genpage.css +++ b/src/wwwroot/css/genpage.css @@ -1218,7 +1218,7 @@ body { height: 2px; background-color: var(--green); } -.controlnet-preview-result img { +.controlnet-preview-result img, .controlnet-preview-result video { max-width: 200px; max-height: 200px; margin-left: 2rem; diff --git a/src/wwwroot/js/genpage/gentab/params.js b/src/wwwroot/js/genpage/gentab/params.js index 5931fcb28..80aa5341d 100644 --- a/src/wwwroot/js/genpage/gentab/params.js +++ b/src/wwwroot/js/genpage/gentab/params.js @@ -1490,10 +1490,25 @@ function controlnetShowPreview() { if (!data.image) { return; } - let imgElem = document.createElement('img'); - imgElem.src = data.image; let resultBox = createDiv(null, 'controlnet-preview-result'); - resultBox.append(imgElem); + let isVideo = isVideoExt(data.image); + if (isVideo) { + let vidElem = document.createElement('video'); + vidElem.loop = true; + vidElem.autoplay = true; + vidElem.muted = true; + vidElem.controls = true; + let sourceObj = document.createElement('source'); + sourceObj.src = data.image; + sourceObj.type = isVideo; + vidElem.append(sourceObj); + resultBox.append(vidElem); + } + else { + let imgElem = document.createElement('img'); + imgElem.src = data.image; + resultBox.append(imgElem); + } clearPreview(); previewArea.append(resultBox); }); From 68af21df9b6e4ba72208ee8372f0db23a70c58a9 Mon Sep 17 00:00:00 2001 From: Juan Treminio Date: Fri, 1 May 2026 19:21:55 -0500 Subject: [PATCH 2/4] resize_type.multiple=2 for videos --- .../ComfyUIBackend/WorkflowGeneratorSteps.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs b/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs index 4b21de73a..9d21e735d 100644 --- a/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs +++ b/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs @@ -1084,6 +1084,17 @@ bool getBestFor(string phrase) if (g.UserInput.Get(T2IParamTypes.ControlNetPreviewOnly)) { g.CurrentMedia = imageNodeActual.WithPath(preprocActual); + if (g.CurrentMedia.DataType == WGNodeData.DT_VIDEO) + { + string resized = g.CreateNode("ResizeImageMaskNode", new JObject() + { + ["input"] = g.CurrentMedia.Path, + ["resize_type"] = "scale to multiple", + ["resize_type.multiple"] = 2, + ["scale_method"] = "lanczos" + }); + g.CurrentMedia = g.CurrentMedia.WithPath([resized, 0]); + } g.CurrentMedia.SaveOutput(g.CurrentVae, g.CurrentAudioVae, id: "9"); g.SkipFurtherSteps = true; return; From 35a63e426e3a1f4e7edccdcf9a608d2430a7165c Mon Sep 17 00:00:00 2001 From: Juan Treminio Date: Fri, 1 May 2026 19:28:53 -0500 Subject: [PATCH 3/4] 8 is better than 2 --- src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs b/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs index 9d21e735d..779a0553c 100644 --- a/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs +++ b/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs @@ -1090,7 +1090,7 @@ bool getBestFor(string phrase) { ["input"] = g.CurrentMedia.Path, ["resize_type"] = "scale to multiple", - ["resize_type.multiple"] = 2, + ["resize_type.multiple"] = 8, ["scale_method"] = "lanczos" }); g.CurrentMedia = g.CurrentMedia.WithPath([resized, 0]); From 3fd0211f016571210cbfc77d3c7e487bcd7123dd Mon Sep 17 00:00:00 2001 From: Juan Treminio Date: Sat, 2 May 2026 00:47:40 -0500 Subject: [PATCH 4/4] multiple=8 for all preprocessors --- .../ComfyUIBackend/WorkflowGeneratorSteps.cs | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs b/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs index 779a0553c..abcf253ef 100644 --- a/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs +++ b/src/BuiltinExtensions/ComfyUIBackend/WorkflowGeneratorSteps.cs @@ -1081,25 +1081,22 @@ bool getBestFor(string phrase) { JArray preprocActual = g.CreatePreprocessor(preprocessor, imageNodeActual); g.NodeHelpers["controlnet_preprocessor"] = $"{preprocActual[0]}"; + imageNodeActual = imageNodeActual.WithPath(preprocActual); + string multipleOf8 = g.CreateNode("ResizeImageMaskNode", new JObject() + { + ["input"] = imageNodeActual.Path, + ["resize_type"] = "scale to multiple", + ["resize_type.multiple"] = 8, + ["scale_method"] = "lanczos" + }); + imageNodeActual = imageNodeActual.WithPath([multipleOf8, 0]); if (g.UserInput.Get(T2IParamTypes.ControlNetPreviewOnly)) { - g.CurrentMedia = imageNodeActual.WithPath(preprocActual); - if (g.CurrentMedia.DataType == WGNodeData.DT_VIDEO) - { - string resized = g.CreateNode("ResizeImageMaskNode", new JObject() - { - ["input"] = g.CurrentMedia.Path, - ["resize_type"] = "scale to multiple", - ["resize_type.multiple"] = 8, - ["scale_method"] = "lanczos" - }); - g.CurrentMedia = g.CurrentMedia.WithPath([resized, 0]); - } + g.CurrentMedia = imageNodeActual; g.CurrentMedia.SaveOutput(g.CurrentVae, g.CurrentAudioVae, id: "9"); g.SkipFurtherSteps = true; return; } - imageNodeActual = imageNodeActual.WithPath(preprocActual); } else if (g.UserInput.Get(T2IParamTypes.ControlNetPreviewOnly)) {