From 2eedcd882e4f26f21633ffffe6aa4dc02765f2e7 Mon Sep 17 00:00:00 2001 From: Omkar Nadkarni Date: Wed, 8 Jul 2026 16:37:30 +0100 Subject: [PATCH 1/3] Added Hunyuan 1.5 T2V and I2V support. Image2Video model takes a single image as input and generates video sequence. - Noise latents need to be resized according to model target size. similarly the input image is also resized to match the latents. - The model also does not take height, width and callback_on_end parameters in its __call__ function so this kwargs are omitted in base driver by checking the signature of the function. - The output latents are finally resized to match the original latent shape in the decode function. --- docs/nodes/pipeline_builder.md | 2 +- .../latent_pipeline_drivers/base_driver.py | 14 +- .../latent_pipeline_drivers/driver_factory.py | 5 + .../hunyuan_video1_5.py | 134 ++++++++++++++++++ .../hunyuan_video1_5_i2v.py | 118 +++++++++++++++ .../parameters/pipelinetype_parameters.py | 25 ++++ .../parameters/providers.py | 1 + ...hunyuan_video1_5_i2v_runtime_parameters.py | 99 +++++++++++++ .../hunyuan_video1_5_runtime_parameters.py | 61 ++++++++ .../runtime_params_registry.py | 8 ++ .../hunyuan_video1_5_i2v_parameters.py | 76 ++++++++++ .../hunyuan_video1_5_parameters.py | 74 ++++++++++ 12 files changed, 612 insertions(+), 5 deletions(-) create mode 100644 modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py create mode 100644 modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py create mode 100644 modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_i2v_runtime_parameters.py create mode 100644 modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_runtime_parameters.py create mode 100644 modular_diffusion_nodes_library/standard_parameters/hunyuan_video1_5_i2v_parameters.py create mode 100644 modular_diffusion_nodes_library/standard_parameters/hunyuan_video1_5_parameters.py diff --git a/docs/nodes/pipeline_builder.md b/docs/nodes/pipeline_builder.md index 51e24d5..95e08d3 100644 --- a/docs/nodes/pipeline_builder.md +++ b/docs/nodes/pipeline_builder.md @@ -38,7 +38,7 @@ Category: `ModularDiffusion/Pipeline` | Name | Type | Notes | | --- | --- | --- | -| `provider` | choice | `Flux`, `Flux2`, `Stable Diffusion`, `Stable Diffusion 3`, `Qwen`, `Z-Image`, `LTX`, `LTX2`, `WAN`. Changing this swaps every parameter below. | +| `provider` | choice | `Flux`, `Flux2`, `Stable Diffusion`, `Stable Diffusion 3`, `Qwen`, `Z-Image`, `HunyuanVideo 1.5`, `LTX`, `LTX2`, `WAN`. Changing this swaps every parameter below. | | `pipeline_type` | choice | Per-provider pipeline class (e.g. `FluxPipeline`, `WanImageToVideoPipeline`). Determines what the pipeline can do. | | `` | HF repo picker | Hugging Face repo ID. Diffusers-format only — single-file `.safetensors` checkpoints are not loaded directly. | diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/base_driver.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/base_driver.py index 08563e8..5c611b9 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/base_driver.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/base_driver.py @@ -1,3 +1,4 @@ +import inspect from abc import ABC, abstractmethod from typing import Any, ClassVar, TypeVar @@ -338,15 +339,20 @@ def denoise_latent( latents = self.prepare_input_latent(latents, source_shape) kwargs["latents"] = latents - kwargs.setdefault("height", source_shape[-2]) - kwargs.setdefault("width", source_shape[-1]) - generator = kwargs.pop("generator", generator_state.to_generator()) + # check that the pipeline supports the kwargs we are passing in + pipe_call_params = inspect.signature(pipe.__call__).parameters + if "height" in pipe_call_params: + kwargs.setdefault("height", source_shape[-2]) + if "width" in pipe_call_params: + kwargs.setdefault("width", source_shape[-1]) + if "callback_on_step_end" in pipe_call_params: + kwargs["callback_on_step_end"] = callback + generator = kwargs.pop("generator", generator_state.to_generator()) pipe_kwargs: dict[str, Any] = { **kwargs, "output_type": "latent", "num_inference_steps": num_inference_steps, - "callback_on_step_end": callback, "generator": generator, } diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py index 3d40de8..6d37fc6 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py @@ -18,6 +18,9 @@ from modular_diffusion_nodes_library.latent_pipeline_drivers.stable_diffusion_xl import ( StableDiffusionXLLatentPipelineDriver, ) +from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5 import HunyuanVideo15TextToVideoLatentPipelineDriver +from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5_i2v import HunyuanVideo15ImageToVideoLatentPipelineDriver + from modular_diffusion_nodes_library.latent_pipeline_drivers.wan import WanTextToVideoLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.wan_animate import WanAnimateLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.wan_i2v import WanImageToVideoLatentPipelineDriver @@ -27,6 +30,8 @@ # Maps pipeline class name prefix to the corresponding driver class. _DRIVER_REGISTRY: dict[str, type[LatentPipelineDriver]] = { "FluxFillPipeline": FluxFillLatentPipelineDriver, + "HunyuanVideo15Pipeline": HunyuanVideo15TextToVideoLatentPipelineDriver, + "HunyuanVideo15ImageToVideoPipeline": HunyuanVideo15ImageToVideoLatentPipelineDriver, "FluxKontextPipeline": FluxKontextLatentPipelineDriver, "FluxPipeline": FluxLatentPipelineDriver, "Flux2Pipeline": Flux2LatentPipelineDriver, diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py new file mode 100644 index 0000000..25546b9 --- /dev/null +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py @@ -0,0 +1,134 @@ +import logging +from typing import Any, ClassVar, override + +import torch # type: ignore[reportMissingImports] +from diffusers.modular_pipelines.hunyuan_video1_5 import ( # type: ignore[reportMissingImports] + HunyuanVideo15AutoBlocks, +) +from diffusers.modular_pipelines.hunyuan_video1_5.before_denoise import ( # type: ignore[reportMissingImports] + HunyuanVideo15PrepareLatentsStep, +) +from diffusers.modular_pipelines.hunyuan_video1_5.decoders import ( # type: ignore[reportMissingImports] + HunyuanVideo15VaeDecoderStep, +) +from diffusers.modular_pipelines.modular_pipeline import ModularPipeline # type: ignore[reportMissingImports] +from diffusers.pipelines.pipeline_utils import DiffusionPipeline # type: ignore[reportMissingImports] + +from modular_diffusion_nodes_library.artifact_utils.inpaint_mask_artifact import InpaintMaskArtifact +from modular_diffusion_nodes_library.artifact_utils.latent_artifact import LatentArtifact +from modular_diffusion_nodes_library.latent_pipeline_drivers.base_driver import LatentPipelineDriver +from modular_diffusion_nodes_library.latent_pipeline_drivers.driver_types import ( + DecodeResult, + GeneratorState, + ImageMedia, + VideoMedia, +) + +logger = logging.getLogger("modular_diffusers_nodes_library") + + +class HunyuanVideo15TextToVideoLatentPipelineDriver(LatentPipelineDriver): + produces_video: ClassVar[bool] = True + # EXAMPLE_DOC_STRING in pipeline_hunyuan_video1_5.py specifies fps=15 + video_fps: ClassVar[int] = 15 + + def __init__(self, pipe: DiffusionPipeline): + super().__init__(pipe) + + @classmethod + @override + def can_make_control_pipe_from_standard(cls, control_net_model_lists: list[str] | str | None) -> bool: + return False + + @override + def _create_modular_pipe(self) -> ModularPipeline: + return HunyuanVideo15AutoBlocks().init_pipeline() + + @override + def _extract_latents_from_output(self, pipe_output: Any) -> torch.Tensor: + """HunyuanVideo pipelines return video frames under `.frames` instead of `.images`.""" + return pipe_output.frames + + @override + def create_noise_latent(self, source_shape: tuple[int, ...], generator_state: GeneratorState) -> LatentArtifact: + generator = generator_state.to_generator() + num_frames, height, width = source_shape[-3], source_shape[-2], source_shape[-1] + prepare_latents = HunyuanVideo15PrepareLatentsStep() + output_state = self._call_block( + prepare_latents, + height=height, + width=width, + num_frames=num_frames, + num_videos_per_prompt=1, + generator=generator, + batch_size=1, + ) + latents = output_state.get("latents") + return self._make_latent_artifact( + latents, + source_shape=source_shape, + meta=GeneratorState.from_generator(generator).as_meta(), + ) + + @override + def decode_latent(self, latent: LatentArtifact) -> DecodeResult: + """Decode a 5-D HunyuanVideo latent and return the video frames.""" + device, dtype = self._get_device_and_type() + latents = latent.to_torch(device=device, dtype=dtype) + vae_decoder_step = HunyuanVideo15VaeDecoderStep() + output_state = self._call_block(vae_decoder_step, latents=latents, output_type="pil") + video_frames = output_state.get("videos")[0] + return video_frames + + @override + def encode_media(self, media: ImageMedia | VideoMedia, generator_state: GeneratorState) -> LatentArtifact: + raise NotImplementedError( + f"Pipeline '{self.pipe.__class__.__name__}' does not support media encoding. " + "This is a text-to-video pipeline." + ) + + @override + def add_noise_to_latent( + self, + latent: LatentArtifact, + generator_state: GeneratorState, + num_inference_steps: int, + strength: float, + ) -> LatentArtifact: + raise NotImplementedError( + f"Pipeline '{self.pipe.__class__.__name__}' does not support img2img noise injection." + ) + + @override + def denoise_latent( + self, + latent: LatentArtifact | InpaintMaskArtifact, + num_inference_steps: int, + generator_state: GeneratorState, + callback: Any = None, + start_step: int = 0, + end_step: int = -1, + return_fully_denoised: bool = False, + **kwargs: Any, + ) -> LatentArtifact: + update_kwargs = kwargs.copy() + update_kwargs.pop("media_gen_conditioning", None) + + guidance_scale = update_kwargs.pop("guidance_scale", None) + if guidance_scale is not None: + self._pipe.guider.guidance_scale = guidance_scale + + update_kwargs["num_frames"] = latent.source_shape[-3] + + # HunyuanVideo15Pipeline.__call__ has no callback_on_step_end parameter, + # so execution status, live preview and mid-run cancellation are not available for this pipeline. + return super().denoise_latent( + latent, + num_inference_steps, + generator_state=generator_state, + callback=callback, + start_step=start_step, + end_step=end_step, + return_fully_denoised=return_fully_denoised, + **update_kwargs, + ) diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py new file mode 100644 index 0000000..b6f30b7 --- /dev/null +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py @@ -0,0 +1,118 @@ +import logging +from typing import Any, ClassVar, override + +from PIL.Image import Image, Resampling + +from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5 import HunyuanVideo15TextToVideoLatentPipelineDriver +from modular_diffusion_nodes_library.artifact_utils.inpaint_mask_artifact import InpaintMaskArtifact +from modular_diffusion_nodes_library.artifact_utils.latent_artifact import LatentArtifact +from modular_diffusion_nodes_library.latent_pipeline_drivers.driver_types import ( + DecodeResult, + GeneratorState, +) +from modular_diffusion_nodes_library.parameters.media_gen_conditioning.conditioning_payload import normalize_to_payloads +from modular_diffusion_nodes_library.utils.conditioning_utils import ( + ConditioningMode, + MediaGenConditioningKey, + resolve_conditioning_image, + resolve_frame_index, +) + +logger = logging.getLogger("modular_diffusers_nodes_library") +class HunyuanVideo15ImageToVideoLatentPipelineDriver(HunyuanVideo15TextToVideoLatentPipelineDriver): + # EXAMPLE_DOC_STRING in pipeline_hunyuan_video1_5_image2video.py specifies fps=24 + video_fps: ClassVar[int] = 24 + + @override + def create_noise_latent(self, source_shape: tuple[int, ...], generator_state: GeneratorState) -> LatentArtifact: + width, height = self.get_resize_dimensions(source_shape[-1], source_shape[-2]) + resized_source_shape = (*source_shape[:-2], height, width) + resized_output = super().create_noise_latent(resized_source_shape, generator_state) + return self._make_latent_artifact( + resized_output.to_torch(), + source_shape=source_shape, + upstream=resized_output, + ) + + @override + def decode_latent(self, latent: LatentArtifact) -> DecodeResult: + frames = super().decode_latent(latent) + source_shape = latent.source_shape + output_frames = [frame.resize((source_shape[-1], source_shape[-2]), Resampling.LANCZOS) for frame in frames] # type: ignore[reportGeneralTypeIssues] + return output_frames + + @override + def denoise_latent( + self, + latent: LatentArtifact | InpaintMaskArtifact, + num_inference_steps: int, + generator_state: GeneratorState, + callback: Any = None, + start_step: int = 0, + end_step: int = -1, + return_fully_denoised: bool = False, + **kwargs: Any, + ) -> LatentArtifact: + update_kwargs = kwargs.copy() + payloads = normalize_to_payloads(update_kwargs.pop(MediaGenConditioningKey.OUTPUT, None)) + + guidance_scale = update_kwargs.pop("guidance_scale", None) + if guidance_scale is not None: + self._pipe.guider.guidance_scale = guidance_scale + + if payloads is not None: + num_frames = latent.source_shape[-3] + for payload in payloads: + if payload.mode is ConditioningMode.IMAGE: + for entry in payload.entries: + image = resolve_conditioning_image(entry.artifact) + image = self.preprocess_image(image, latent.source_shape[-1], latent.source_shape[-2]) + frame_index = resolve_frame_index(entry.frame_index, num_frames) + if frame_index == 0: + update_kwargs["image"] = image + else: + msg = ( + f"Attempted to build HunyuanVideo I2V conditioning. " + f"Failed with frame_index={frame_index} because only frame_index=0 is supported." + ) + raise ValueError(msg) + else: + msg = ( + f"Failed to build HunyuanVideo I2V conditioning because " + f"mode '{payload.mode.value}' is unsupported." + ) + raise ValueError(msg) + + if "image" not in update_kwargs: + raise ValueError( + f"{self.driver_namespace}: HunyuanVideo I2V requires a first-frame image (frame_index=0) " + "via media_gen_conditioning." + ) + + update_kwargs["num_frames"] = latent.source_shape[-3] + # height/width are NOT injected — HunyuanVideo I2V derives them from the image internally. + + return super().denoise_latent( + latent, + num_inference_steps, + generator_state=generator_state, + callback=callback, + start_step=start_step, + end_step=end_step, + return_fully_denoised=return_fully_denoised, + **update_kwargs, + ) + + + def get_resize_dimensions(self, width: int, height: int) -> tuple[int, int]: + """Calculate the resize dimensions for a given width and height.""" + target_size = self.pipe.transformer.config.target_size if getattr(self.pipe, "transformer", None) else 640 + height, width = self.pipe.video_processor.calculate_default_height_width(height=height, width=width, target_size=target_size) + return width, height + + def preprocess_image(self, image: Image, width: int, height: int) -> Image: + """Preprocess a PIL image for WAN i2v encoding.""" + # Automatically resize image based on model capabilities + resized_width, resized_height = self.get_resize_dimensions(width, height) + image = image.resize((resized_width, resized_height)) + return image \ No newline at end of file diff --git a/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py b/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py index 8efe6c3..7e5d2ac 100644 --- a/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py +++ b/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py @@ -27,6 +27,12 @@ from modular_diffusion_nodes_library.standard_parameters.flux_parameters import ( FluxPipelineParameters, ) +from modular_diffusion_nodes_library.standard_parameters.hunyuan_video1_5_i2v_parameters import ( + HunyuanVideo15ImageToVideoPipelineParameters, +) +from modular_diffusion_nodes_library.standard_parameters.hunyuan_video1_5_parameters import ( + HunyuanVideo15PipelineParameters, +) from modular_diffusion_nodes_library.standard_parameters.ltx2_parameters import LTX2PipelineParameters from modular_diffusion_nodes_library.standard_parameters.ltx_parameters import ( LTXPipelineParameters, @@ -209,6 +215,24 @@ def get_pipeline_type_dict(cls) -> dict[str, type[ModularDiffusionPipelineTypePi } +class LatentHunyuanVideo15PipelineTypeParameters(LatentPipelineTypeParameters): + @property + def pipeline_type_badge_message(self) -> str: + return ( + "- `HunyuanVideo15Pipeline` — Text-to-video generation (Tencent HunyuanVideo 1.5).\n" + "- `HunyuanVideo15ImageToVideoPipeline` — Image-to-video. Requires a dedicated I2V checkpoint. \n\n" + "For first-frame conditioning, connect a Media Gen Conditioning node to the " + "`additional_parameters` input of Generate Media Latents." + ) + + @classmethod + def get_pipeline_type_dict(cls) -> dict[str, type[ModularDiffusionPipelineTypePipelineParameters]]: + return { + "HunyuanVideo15Pipeline": HunyuanVideo15PipelineParameters, + "HunyuanVideo15ImageToVideoPipeline": HunyuanVideo15ImageToVideoPipelineParameters, + } + + class LatentLTX2PipelineTypeParameters(LatentPipelineTypeParameters): @property def pipeline_type_badge_message(self) -> str: @@ -333,6 +357,7 @@ def get_pipeline_type_dict(cls) -> dict[str, type[ModularDiffusionPipelineTypePi MODULAR_PIPELINE_TYPE_PROVIDER_MAP: dict[Provider, type[LatentPipelineTypeParameters]] = { Provider.FLUX: LatentFluxPipelineTypeParameters, Provider.FLUX2: LatentFlux2PipelineTypeParameters, + Provider.HUNYUAN_VIDEO_1_5: LatentHunyuanVideo15PipelineTypeParameters, Provider.LTX: LatentLTXPipelineTypeParameters, Provider.LTX2: LatentLTX2PipelineTypeParameters, Provider.QWEN: LatentQwenPipelineTypeParameters, diff --git a/modular_diffusion_nodes_library/parameters/providers.py b/modular_diffusion_nodes_library/parameters/providers.py index a0c4c81..586a4ce 100644 --- a/modular_diffusion_nodes_library/parameters/providers.py +++ b/modular_diffusion_nodes_library/parameters/providers.py @@ -4,6 +4,7 @@ class Provider(StrEnum): FLUX = "Flux" FLUX2 = "Flux2" + HUNYUAN_VIDEO_1_5 = "HunyuanVideo 1.5" LTX = "LTX" LTX2 = "LTX2" QWEN = "Qwen" diff --git a/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_i2v_runtime_parameters.py b/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_i2v_runtime_parameters.py new file mode 100644 index 0000000..fc9def2 --- /dev/null +++ b/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_i2v_runtime_parameters.py @@ -0,0 +1,99 @@ +import logging +from typing import ClassVar + +from griptape_nodes.exe_types.core_types import Parameter +from griptape_nodes.exe_types.node_types import BaseNode + +from modular_diffusion_nodes_library.parameters.media_gen_conditioning.conditioning_layout import ( + PRESET_FIRST, + MediaGenConditioningConfig, + PresetCatalogImageConfig, +) +from modular_diffusion_nodes_library.runtime_parameters.conditioning_runtime_parameter import ( + MediaGenConditioningRuntimeParameter, +) +from modular_diffusion_nodes_library.runtime_parameters.runtime_parameters import ( + DiffusionPipelineRuntimeParameters, +) +from modular_diffusion_nodes_library.utils.conditioning_utils import ConditioningMode + +logger = logging.getLogger("diffusers_nodes_library") + + +class HunyuanVideo15ImageToVideoPipelineRuntimeParameters(DiffusionPipelineRuntimeParameters): + # HunyuanVideo I2V conditions only on the first frame (no last-frame support). + CONDITIONING_CONFIG: ClassVar[MediaGenConditioningConfig | None] = MediaGenConditioningConfig( + image=PresetCatalogImageConfig(presets=(PRESET_FIRST,), expose_strength=False), + ) + + def __init__(self, node: BaseNode): + super().__init__(node) + self._media_gen_conditioning_param = MediaGenConditioningRuntimeParameter( + node, + param_name="conditioning_images", + accepted_modes=(ConditioningMode.IMAGE,), + tooltip="First-frame conditioning image for HunyuanVideo I2V, from a Media Gen Conditioning node.", + badge_title="First-frame conditioning image", + badge_message=( + "Connect a **Media Gen Conditioning** node here to supply the start frame " + "for image-to-video generation. Only **image**-mode payloads are accepted.\n\n" + "**Tip:** You can also connect an image directly — without a Media Gen Conditioning node — " + "for single first-frame conditioning (frame index **0**)." + ), + ) + + def _add_input_parameters(self) -> None: + self._node.add_parameter( + Parameter( + name="prompt", + default_value="", + type="str", + tooltip="The prompt or prompts to guide the video generation from the input image.", + ) + ) + self._node.add_parameter( + Parameter( + name="negative_prompt", + default_value="", + type="str", + tooltip="The prompt or prompts not to guide the video generation.", + ) + ) + self._media_gen_conditioning_param.add_input_parameters() + guidance_scale_param = Parameter( + name="guidance_scale", + default_value=7.5, + type="float", + tooltip=( + "Controls how strongly the model follows the text prompt. " + "Higher values produce videos that more closely match the prompt, usually at the expense of quality. " + "Default 7.5 is sourced from the ClassifierFreeGuidance component default." + ), + ) + guidance_scale_param.set_badge( + variant="help", + title="Guidance scale", + message="For recommended values, reset the node to restore the model author's defaults.", + ) + self._node.add_parameter(guidance_scale_param) + + def _remove_input_parameters(self) -> None: + self._media_gen_conditioning_param.remove_input_parameters() + self._node.remove_parameter_element_by_name("prompt") + self._node.remove_parameter_element_by_name("negative_prompt") + self._node.remove_parameter_element_by_name("guidance_scale") + + def _get_pipe_kwargs(self) -> dict: + return { + "prompt": self._node.get_parameter_value("prompt"), + "negative_prompt": self._node.get_parameter_value("negative_prompt"), + "guidance_scale": self._node.get_parameter_value("guidance_scale"), + **self._media_gen_conditioning_param.get_pipe_kwargs(), + } + + def validate_before_node_run(self) -> list[Exception] | None: + errors = super().validate_before_node_run() or [] + conditioning_errors = self._media_gen_conditioning_param.validate_before_node_run() + if conditioning_errors: + errors.extend(conditioning_errors) + return errors or None diff --git a/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_runtime_parameters.py b/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_runtime_parameters.py new file mode 100644 index 0000000..b9271da --- /dev/null +++ b/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_runtime_parameters.py @@ -0,0 +1,61 @@ +import logging + +from griptape_nodes.exe_types.core_types import Parameter +from griptape_nodes.exe_types.node_types import BaseNode + +from modular_diffusion_nodes_library.runtime_parameters.runtime_parameters import ( + DiffusionPipelineRuntimeParameters, +) + +logger = logging.getLogger("diffusers_nodes_library") + + +class HunyuanVideo15PipelineRuntimeParameters(DiffusionPipelineRuntimeParameters): + def __init__(self, node: BaseNode): + super().__init__(node) + + def _add_input_parameters(self) -> None: + self._node.add_parameter( + Parameter( + name="prompt", + default_value="", + type="str", + tooltip="The prompt or prompts to guide the video generation.", + ) + ) + self._node.add_parameter( + Parameter( + name="negative_prompt", + default_value="", + type="str", + tooltip="The prompt or prompts not to guide the video generation.", + ) + ) + guidance_scale_param = Parameter( + name="guidance_scale", + default_value=7.5, + type="float", + tooltip=( + "Controls how strongly the model follows the text prompt. " + "Higher values produce videos that more closely match the prompt, usually at the expense of quality. " + "Default 7.5 is sourced from the ClassifierFreeGuidance component default." + ), + ) + guidance_scale_param.set_badge( + variant="help", + title="Guidance scale", + message="For recommended values, reset the node to restore the model author's defaults.", + ) + self._node.add_parameter(guidance_scale_param) + + def _remove_input_parameters(self) -> None: + self._node.remove_parameter_element_by_name("prompt") + self._node.remove_parameter_element_by_name("negative_prompt") + self._node.remove_parameter_element_by_name("guidance_scale") + + def _get_pipe_kwargs(self) -> dict: + return { + "prompt": self._node.get_parameter_value("prompt"), + "negative_prompt": self._node.get_parameter_value("negative_prompt"), + "guidance_scale": self._node.get_parameter_value("guidance_scale"), + } diff --git a/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py b/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py index e84bf70..753111a 100644 --- a/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py +++ b/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py @@ -19,6 +19,12 @@ from modular_diffusion_nodes_library.runtime_parameters.flux_fill_runtime_parameters import ( FluxFillPipelineRuntimeParameters, ) +from modular_diffusion_nodes_library.runtime_parameters.hunyuan_video1_5_i2v_runtime_parameters import ( + HunyuanVideo15ImageToVideoPipelineRuntimeParameters, +) +from modular_diffusion_nodes_library.runtime_parameters.hunyuan_video1_5_runtime_parameters import ( + HunyuanVideo15PipelineRuntimeParameters, +) from modular_diffusion_nodes_library.runtime_parameters.flux_kontext_runtime_parameters import ( FluxKontextPipelineRuntimeParameters, ) @@ -68,6 +74,8 @@ "FluxKontextPipeline": FluxKontextPipelineRuntimeParameters, "Flux2Pipeline": Flux2PipelineRuntimeParameters, "Flux2KleinPipeline": Flux2KleinPipelineRuntimeParameters, + "HunyuanVideo15Pipeline": HunyuanVideo15PipelineRuntimeParameters, + "HunyuanVideo15ImageToVideoPipeline": HunyuanVideo15ImageToVideoPipelineRuntimeParameters, "LTXPipeline": LTXPipelineRuntimeParameters, "LTX2Pipeline": LTX2PipelineRuntimeParameters, "QwenImagePipeline": QwenPipelineRuntimeParameters, diff --git a/modular_diffusion_nodes_library/standard_parameters/hunyuan_video1_5_i2v_parameters.py b/modular_diffusion_nodes_library/standard_parameters/hunyuan_video1_5_i2v_parameters.py new file mode 100644 index 0000000..26667ce --- /dev/null +++ b/modular_diffusion_nodes_library/standard_parameters/hunyuan_video1_5_i2v_parameters.py @@ -0,0 +1,76 @@ +import logging +from typing import Any + +import diffusers # type: ignore[reportMissingImports] +import torch # type: ignore[reportMissingImports] +from griptape_nodes.exe_types.node_types import BaseNode +from griptape_nodes.exe_types.param_components.huggingface.huggingface_repo_parameter import HuggingFaceRepoParameter + +from modular_diffusion_nodes_library.parameters.modular_pipeline_type_parameters import ( + ModularDiffusionPipelineTypePipelineParameters, +) + +logger = logging.getLogger("modular_diffusers_nodes_library") + + +class HunyuanVideo15ImageToVideoPipelineParameters(ModularDiffusionPipelineTypePipelineParameters): + def __init__(self, node: BaseNode, *, list_all_models: bool = False): + super().__init__(node) + self._model_repo_parameter = HuggingFaceRepoParameter( + node, + repo_ids=[ + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v_distilled", + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v", + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v_distilled", + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v_step_distilled", + ], + parameter_name="model", + list_all_models=list_all_models, + ) + + def add_input_parameters(self) -> None: + self._model_repo_parameter.add_input_parameters() + + def remove_input_parameters(self) -> None: + self._model_repo_parameter.remove_input_parameters() + + def get_config_kwargs(self) -> dict: + return { + "model": self._node.get_parameter_value("model"), + } + + @property + def pipeline_class(self) -> type: + return diffusers.HunyuanVideo15ImageToVideoPipeline # type: ignore[reportAttributeAccessIssue] + + def validate_before_node_run(self) -> list[Exception] | None: + errors = [] + model_errors = self._model_repo_parameter.validate_before_node_run() + if model_errors: + errors.extend(model_errors) + return errors or None + + def get_build_data(self) -> dict[str, Any]: + repo_id, revision = self._model_repo_parameter.get_repo_revision() + return { + "repo_id": repo_id, + "revision": revision, + } + + def requires_device_map(self) -> bool: + # HunyuanVideo 1.5 I2V includes a Qwen2.5-VL-7B text encoder, a SiglipVisionModel + # image encoder, plus the video transformer — total weights exceed 20 GB in + # bfloat16. Using device_map lets accelerate stream each layer directly to the + # right device during loading so we never materialise the full model on CPU RAM. + return True + + @classmethod + def build_pipeline_from_build_data(cls, build_data: dict[str, Any]) -> diffusers.HunyuanVideo15ImageToVideoPipeline: # type: ignore[reportAttributeAccessIssue] + return diffusers.HunyuanVideo15ImageToVideoPipeline.from_pretrained( # type: ignore[reportAttributeAccessIssue] + pretrained_model_name_or_path=build_data["repo_id"], + revision=build_data["revision"], + torch_dtype=torch.bfloat16, + local_files_only=True, + device_map="balanced", + ) diff --git a/modular_diffusion_nodes_library/standard_parameters/hunyuan_video1_5_parameters.py b/modular_diffusion_nodes_library/standard_parameters/hunyuan_video1_5_parameters.py new file mode 100644 index 0000000..0bc88b5 --- /dev/null +++ b/modular_diffusion_nodes_library/standard_parameters/hunyuan_video1_5_parameters.py @@ -0,0 +1,74 @@ +import logging +from typing import Any + +import diffusers # type: ignore[reportMissingImports] +import torch # type: ignore[reportMissingImports] +from griptape_nodes.exe_types.node_types import BaseNode +from griptape_nodes.exe_types.param_components.huggingface.huggingface_repo_parameter import HuggingFaceRepoParameter + +from modular_diffusion_nodes_library.parameters.modular_pipeline_type_parameters import ( + ModularDiffusionPipelineTypePipelineParameters, +) + +logger = logging.getLogger("modular_diffusers_nodes_library") + + +class HunyuanVideo15PipelineParameters(ModularDiffusionPipelineTypePipelineParameters): + def __init__(self, node: BaseNode, *, list_all_models: bool = False): + super().__init__(node) + self._model_repo_parameter = HuggingFaceRepoParameter( + node, + repo_ids=[ + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v", + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v_distilled", + ], + parameter_name="model", + list_all_models=list_all_models, + ) + + def add_input_parameters(self) -> None: + self._model_repo_parameter.add_input_parameters() + + def remove_input_parameters(self) -> None: + self._model_repo_parameter.remove_input_parameters() + + def get_config_kwargs(self) -> dict: + return { + "model": self._node.get_parameter_value("model"), + } + + @property + def pipeline_class(self) -> type: + return diffusers.HunyuanVideo15Pipeline # type: ignore[reportAttributeAccessIssue] + + def validate_before_node_run(self) -> list[Exception] | None: + errors = [] + model_errors = self._model_repo_parameter.validate_before_node_run() + if model_errors: + errors.extend(model_errors) + return errors or None + + def get_build_data(self) -> dict[str, Any]: + repo_id, revision = self._model_repo_parameter.get_repo_revision() + return { + "repo_id": repo_id, + "revision": revision, + } + + def requires_device_map(self) -> bool: + # HunyuanVideo 1.5 includes a Qwen2.5-VL-7B text encoder plus the video + # transformer — total weights exceed 20 GB in bfloat16. Using device_map + # lets accelerate stream each layer directly to the right device during + # loading so we never materialise the full model on CPU RAM. + return True + + @classmethod + def build_pipeline_from_build_data(cls, build_data: dict[str, Any]) -> diffusers.HunyuanVideo15Pipeline: # type: ignore[reportAttributeAccessIssue] + return diffusers.HunyuanVideo15Pipeline.from_pretrained( # type: ignore[reportAttributeAccessIssue] + pretrained_model_name_or_path=build_data["repo_id"], + revision=build_data["revision"], + torch_dtype=torch.bfloat16, + local_files_only=True, + device_map="balanced", + ) From 37a605678059e9b43836c9a7493e1eb4d0d3cf8a Mon Sep 17 00:00:00 2001 From: Omkar Nadkarni Date: Mon, 13 Jul 2026 11:23:54 +0100 Subject: [PATCH 2/3] Implement encode media and add_noise_to_latents for HunyuanVideo 1.5 pipelines. - Updated Claude code skills to implement encode media and add_noise_to_latents for video pipelines. --- .github/skills/add-modular-pipeline/SKILL.md | 5 +- .../hunyuan_video1_5.py | 46 ++++++++++++++++--- .../hunyuan_video1_5_i2v.py | 26 ++++++++++- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/.github/skills/add-modular-pipeline/SKILL.md b/.github/skills/add-modular-pipeline/SKILL.md index 6ab05c6..d361481 100644 --- a/.github/skills/add-modular-pipeline/SKILL.md +++ b/.github/skills/add-modular-pipeline/SKILL.md @@ -94,6 +94,8 @@ The latent shape & space contract is documented in the `LatentPipelineDriver` do - **Public latents are unpacked**: 4-D `[B, C, H/vae, W/vae]` for image, 5-D `[B, C, T_lat, H/vae, W/vae]` for video. No model-specific sequence packing on the public surface. - **Public methods exchange `LatentArtifact`**, never raw tensors. The four forwardable methods (`encode_media`, `decode_latent`, `create_noise_latent`, `add_noise_to_latent`) return `LatentArtifact` (or `DecodeResult` for `decode_latent`) and accept input artifacts / media dataclasses. Build outputs via `_make_latent_artifact(...)`. +- **`encode_media` for video drivers**: Image-only drivers raise `NotImplementedError` for `VideoMedia`; video-only (T2V) drivers raise for `ImageMedia`. "Vice versa" does NOT mean "video drivers raise for all media" — T2V drivers MUST implement `encode_media` for `VideoMedia` when the pipeline supports video-to-video (V2V). I2V drivers SHOULD additionally implement `encode_media` for `ImageMedia` to enable latent composition workflows. Check the Variants checklist (V2V item) before deciding. +- **`add_noise_to_latent` for video drivers**: This method is NOT image-only. V2V workflows (encoding a source video, adding noise at a given `strength`, then denoising) require a working `add_noise_to_latent` on video drivers. For flow-matching schedulers (`FlowMatchEulerDiscreteScheduler`) use `scheduler.scale_noise`; for DDPM-family schedulers use `scheduler.add_noise`. Only raise `NotImplementedError` when you have verified the underlying pipeline has no V2V support and have cited the diffusers source proving it. - **Forwardable signature is enforced**: those four methods MUST match [`FORWARDABLE_METHOD_POSITIONAL`](../../../modular_diffusion_nodes_library/latent_pipeline_drivers/_base_driver_forwardable_signature.py) exactly — same names, same order, no extra positional or kw-only parameters, no `*args`/`**kwargs`. `__init_subclass__` raises `TypeError` at import time otherwise. Route driver-specific tunables through the driver-namespaced sub-bag on `LatentArtifact.meta` (`META_DRIVER_KEY`, `read_driver_meta` in [`driver_types.py`](../../../modular_diffusion_nodes_library/latent_pipeline_drivers/driver_types.py); SDXL `_KIND_META_KEY` is the canonical precedent). - **Public latents are normalised** (~N(0, 1)). If the VAE config publishes per-channel `latents_mean` / `latents_std`, apply whitening `(z - mean) / std` inside `encode_media` and the inverse inside `decode_latent`. If it publishes only scalar `scaling_factor` (and optional `shift_factor`), the standard `(z - shift) * scaling` transform is sufficient — no whitening. Verify on the actual VAE config of the model you are adding; do not assume from family name. Mirror the closest existing driver. - **`source_shape` rides on the artifact / media dataclass** (pixel-space), not as a separate parameter. `LatentArtifact.source_shape`, `ImageMedia.source_shape`, `VideoMedia.source_shape`, `MaskMedia.source_shape` are the only sources of truth. Drivers translate to latent-space dims internally. @@ -223,7 +225,7 @@ Produce a plan in this format: | encode_prompt | ModularPipeline.sub_blocks["text_encoder"] (or override) | exists at ...; declare extra required inputs (e.g. `prompt_2`, image conditioning) if the block needs them | enumerate every `required=True` `InputParam` | | decode_latent | ModularPipeline.sub_blocks["decode"] | exists at ... | `latents` ← `latent.to_torch(device, dtype)`; `output_type="pil"` | | create_noise_latent | ModularPipeline PrepareLatents step | ... | `height`/`width` ← `source_shape[-2:]`; `batch_size=1`; `num_images_per_prompt=1`; `generator` ← `generator_state.to_generator()` | -| add_noise_to_latent | ModularPipeline Img2Img steps | ... | enumerate every `required=True` `InputParam` (see Modular-Blocks-First rule); e.g. `latents` ← `self.create_noise_latent(...).to_torch(device, dtype)`, `image_latents` ← `latent.to_torch(device, dtype)`, `batch_size` ← `image_latents.shape[0]`, `dtype` ← `_get_device_and_type()[1]`, `generator` ← `generator_state.to_generator()` | +| add_noise_to_latent | ModularPipeline Img2Img / Vid2Vid noise injection steps | ... | enumerate every `required=True` `InputParam` (see Modular-Blocks-First rule); e.g. `latents` ← `self.create_noise_latent(...).to_torch(device, dtype)`, `image_latents` ← `latent.to_torch(device, dtype)`, `batch_size` ← `image_latents.shape[0]`, `dtype` ← `_get_device_and_type()[1]`, `generator` ← `generator_state.to_generator()`. For video pipelines: `image_latents` ← encoded video latent; noise API is `scheduler.scale_noise` (flow-matching) or `scheduler.add_noise` (DDPM) — verify against the scheduler class. | | denoise_latent | super() → DiffusionPipeline.__call__ | three blockers, see SKILL | n/a (delegated) | ## Provider classification (must match the Rule 1, Axis B approval) @@ -264,6 +266,7 @@ State whether this pipeline class gets a NEW runtime-parameters class or reuses - ControlNet supported? [yes/no] — if yes, plan to apply Pattern A from `/add-pipeline-variants` - Inpaint supported? [yes/no] — if yes, plan to set `_inpaint_pipeline_class` (Pattern B) - Runtime pipe-swap needed (e.g. conditional variant)? [yes/no] — if yes, plan Pattern C +- **Video-to-video (V2V) supported?** [yes/no] — **video drivers only**. Determines whether `encode_media(VideoMedia)` and `add_noise_to_latent` need real implementations instead of `NotImplementedError`. To answer yes: (a) the VAE must accept `[B, C, T, H, W]` multi-frame input, and (b) the scheduler must expose `scale_noise` or `add_noise` for video latents. Cite the diffusers source for both. If no, raise `NotImplementedError` with an inline comment citing the source that confirms V2V is unsupported. ## Runtime parameter defaults and tooltips (Rule 4 — every cell needs an upstream citation) | Param | Default value | Default source | Tooltip text source | diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py index 25546b9..95dfca1 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py @@ -7,6 +7,7 @@ ) from diffusers.modular_pipelines.hunyuan_video1_5.before_denoise import ( # type: ignore[reportMissingImports] HunyuanVideo15PrepareLatentsStep, + HunyuanVideo15SetTimestepsStep, ) from diffusers.modular_pipelines.hunyuan_video1_5.decoders import ( # type: ignore[reportMissingImports] HunyuanVideo15VaeDecoderStep, @@ -82,10 +83,25 @@ def decode_latent(self, latent: LatentArtifact) -> DecodeResult: @override def encode_media(self, media: ImageMedia | VideoMedia, generator_state: GeneratorState) -> LatentArtifact: - raise NotImplementedError( - f"Pipeline '{self.pipe.__class__.__name__}' does not support media encoding. " - "This is a text-to-video pipeline." - ) + if isinstance(media, ImageMedia): + raise NotImplementedError( + f"Pipeline '{self.pipe.__class__.__name__}' does not support image encoding. " + "Use a video input instead." + ) + device, dtype = self._get_device_and_type() + vae = self.pipe.vae + frame_tensors = [ + self.pipe.video_processor.preprocess(frame) + for frame in media.frames + ] + video_tensor = torch.stack([t.squeeze(0) for t in frame_tensors], dim=0) # [T, C, H, W] + video_tensor = video_tensor.permute(1, 0, 2, 3).unsqueeze(0) # [1, C, T, H, W] + video_tensor = video_tensor.to(device=device, dtype=vae.dtype) + generator = generator_state.to_generator() + with torch.no_grad(): + latents = vae.encode(video_tensor).latent_dist.sample(generator=generator) + latents = latents.to(dtype=dtype) * vae.config.scaling_factor + return self._make_latent_artifact(latents, source_shape=media.source_shape) @override def add_noise_to_latent( @@ -95,8 +111,26 @@ def add_noise_to_latent( num_inference_steps: int, strength: float, ) -> LatentArtifact: - raise NotImplementedError( - f"Pipeline '{self.pipe.__class__.__name__}' does not support img2img noise injection." + device, dtype = self._get_device_and_type() + source_shape = latent.source_shape + latents = latent.to_torch(device=device, dtype=dtype) + timesteps_state = self._call_block(HunyuanVideo15SetTimestepsStep(), num_inference_steps=num_inference_steps) + timesteps = timesteps_state.get("timesteps") + if timesteps is None or len(timesteps) == 0: + raise ValueError("HunyuanVideo15SetTimestepsStep did not return valid timesteps.") + noise_artifact = self.create_noise_latent(source_shape, generator_state) + noise = noise_artifact.to_torch(device=device, dtype=dtype) + noise_generator_state = GeneratorState.from_artifact(noise_artifact) or generator_state + init_timestep = min(num_inference_steps * strength, num_inference_steps) + t_start = int(max(num_inference_steps - init_timestep, 0)) + latent_timestep = timesteps[t_start * self.pipe.scheduler.order :][:1] + with torch.no_grad(): + result = self.pipe.scheduler.scale_noise(latents, latent_timestep, noise) + return self._make_latent_artifact( + result, + source_shape=source_shape, + upstream=latent, + meta=noise_generator_state.as_meta(), ) @override diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py index b6f30b7..e8f065c 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py @@ -1,15 +1,21 @@ import logging from typing import Any, ClassVar, override +import torch # type: ignore[reportMissingImports] from PIL.Image import Image, Resampling +from diffusers.modular_pipelines.hunyuan_video1_5.encoders import ( # type: ignore[reportMissingImports] + HunyuanVideo15VaeEncoderStep, +) -from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5 import HunyuanVideo15TextToVideoLatentPipelineDriver from modular_diffusion_nodes_library.artifact_utils.inpaint_mask_artifact import InpaintMaskArtifact from modular_diffusion_nodes_library.artifact_utils.latent_artifact import LatentArtifact from modular_diffusion_nodes_library.latent_pipeline_drivers.driver_types import ( DecodeResult, GeneratorState, + ImageMedia, + VideoMedia, ) +from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5 import HunyuanVideo15TextToVideoLatentPipelineDriver from modular_diffusion_nodes_library.parameters.media_gen_conditioning.conditioning_payload import normalize_to_payloads from modular_diffusion_nodes_library.utils.conditioning_utils import ( ConditioningMode, @@ -23,6 +29,24 @@ class HunyuanVideo15ImageToVideoLatentPipelineDriver(HunyuanVideo15TextToVideoLa # EXAMPLE_DOC_STRING in pipeline_hunyuan_video1_5_image2video.py specifies fps=24 video_fps: ClassVar[int] = 24 + @override + def encode_media(self, media: ImageMedia | VideoMedia, generator_state: GeneratorState) -> LatentArtifact: + if isinstance(media, VideoMedia): + resized_width, resized_height = self.get_resize_dimensions(media.source_shape[-1], media.source_shape[-2]) + resized_frames = [frame.resize((resized_width, resized_height)) for frame in media.frames] + preprocessed = VideoMedia(frames=resized_frames, source_shape=media.source_shape) + return super().encode_media(preprocessed, generator_state) + output_state = self._call_block( + HunyuanVideo15VaeEncoderStep(), + image=media.image, + height=media.source_shape[-2], + width=media.source_shape[-1], + ) + return self._make_latent_artifact( + self._get_required(output_state, "image_latents", torch.Tensor), + source_shape=media.source_shape, + ) + @override def create_noise_latent(self, source_shape: tuple[int, ...], generator_state: GeneratorState) -> LatentArtifact: width, height = self.get_resize_dimensions(source_shape[-1], source_shape[-2]) From cff3b0ffcc00499a2a43c51a83f55eea944ef3b4 Mon Sep 17 00:00:00 2001 From: Omkar Nadkarni Date: Tue, 14 Jul 2026 16:46:01 +0100 Subject: [PATCH 3/3] Address code review comments. Improve badge and tooltip message. Improve skills documentation. Create blocks for encode_media and add_noise_to_latents. Fix formatting issues. --- .github/skills/add-modular-pipeline/SKILL.md | 4 +- .../latent_pipeline_drivers/driver_factory.py | 9 +- .../hunyuan_video1_5.py | 151 ++++++++++++++---- .../hunyuan_video1_5_i2v.py | 15 +- .../parameters/pipelinetype_parameters.py | 4 +- ...hunyuan_video1_5_i2v_runtime_parameters.py | 5 +- .../hunyuan_video1_5_runtime_parameters.py | 3 +- .../runtime_params_registry.py | 12 +- 8 files changed, 149 insertions(+), 54 deletions(-) diff --git a/.github/skills/add-modular-pipeline/SKILL.md b/.github/skills/add-modular-pipeline/SKILL.md index d361481..0547e67 100644 --- a/.github/skills/add-modular-pipeline/SKILL.md +++ b/.github/skills/add-modular-pipeline/SKILL.md @@ -94,8 +94,8 @@ The latent shape & space contract is documented in the `LatentPipelineDriver` do - **Public latents are unpacked**: 4-D `[B, C, H/vae, W/vae]` for image, 5-D `[B, C, T_lat, H/vae, W/vae]` for video. No model-specific sequence packing on the public surface. - **Public methods exchange `LatentArtifact`**, never raw tensors. The four forwardable methods (`encode_media`, `decode_latent`, `create_noise_latent`, `add_noise_to_latent`) return `LatentArtifact` (or `DecodeResult` for `decode_latent`) and accept input artifacts / media dataclasses. Build outputs via `_make_latent_artifact(...)`. -- **`encode_media` for video drivers**: Image-only drivers raise `NotImplementedError` for `VideoMedia`; video-only (T2V) drivers raise for `ImageMedia`. "Vice versa" does NOT mean "video drivers raise for all media" — T2V drivers MUST implement `encode_media` for `VideoMedia` when the pipeline supports video-to-video (V2V). I2V drivers SHOULD additionally implement `encode_media` for `ImageMedia` to enable latent composition workflows. Check the Variants checklist (V2V item) before deciding. -- **`add_noise_to_latent` for video drivers**: This method is NOT image-only. V2V workflows (encoding a source video, adding noise at a given `strength`, then denoising) require a working `add_noise_to_latent` on video drivers. For flow-matching schedulers (`FlowMatchEulerDiscreteScheduler`) use `scheduler.scale_noise`; for DDPM-family schedulers use `scheduler.add_noise`. Only raise `NotImplementedError` when you have verified the underlying pipeline has no V2V support and have cited the diffusers source proving it. +- **`encode_media` for image and video drivers**: Prioritise modular implementation — verify whether a VAE encoder block exists for the pipeline before writing inline encode logic. Image-only drivers implement `encode_media` for `ImageMedia` and raise `NotImplementedError` for `VideoMedia`; video drivers raise for `ImageMedia`. Check the Variants checklist (V2V item) before deciding. +- **`add_noise_to_latent` for image and video drivers**: I2I and V2V workflows (encoding a source video, adding noise at a given `strength`, then denoising) require a working `add_noise_to_latent`. Prioritise modular implementation and verify whether relevant blocks exist to achieve this flow. If no modular blocks exist, for flow-matching schedulers (`FlowMatchEulerDiscreteScheduler`) use `scheduler.scale_noise`; for DDPM-family schedulers use `scheduler.add_noise`. Only raise `NotImplementedError` when you have verified the underlying pipeline has no I2I for image pipelines or V2V support for video pipelines and have cited the diffusers source proving it. - **Forwardable signature is enforced**: those four methods MUST match [`FORWARDABLE_METHOD_POSITIONAL`](../../../modular_diffusion_nodes_library/latent_pipeline_drivers/_base_driver_forwardable_signature.py) exactly — same names, same order, no extra positional or kw-only parameters, no `*args`/`**kwargs`. `__init_subclass__` raises `TypeError` at import time otherwise. Route driver-specific tunables through the driver-namespaced sub-bag on `LatentArtifact.meta` (`META_DRIVER_KEY`, `read_driver_meta` in [`driver_types.py`](../../../modular_diffusion_nodes_library/latent_pipeline_drivers/driver_types.py); SDXL `_KIND_META_KEY` is the canonical precedent). - **Public latents are normalised** (~N(0, 1)). If the VAE config publishes per-channel `latents_mean` / `latents_std`, apply whitening `(z - mean) / std` inside `encode_media` and the inverse inside `decode_latent`. If it publishes only scalar `scaling_factor` (and optional `shift_factor`), the standard `(z - shift) * scaling` transform is sufficient — no whitening. Verify on the actual VAE config of the model you are adding; do not assume from family name. Mirror the closest existing driver. - **`source_shape` rides on the artifact / media dataclass** (pixel-space), not as a separate parameter. `LatentArtifact.source_shape`, `ImageMedia.source_shape`, `VideoMedia.source_shape`, `MaskMedia.source_shape` are the only sources of truth. Drivers translate to latent-space dims internally. diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py index 6d37fc6..ada9cce 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py @@ -8,6 +8,12 @@ from modular_diffusion_nodes_library.latent_pipeline_drivers.flux2_klein import Flux2KleinLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.flux_fill import FluxFillLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.flux_kontext import FluxKontextLatentPipelineDriver +from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5 import ( + HunyuanVideo15TextToVideoLatentPipelineDriver, +) +from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5_i2v import ( + HunyuanVideo15ImageToVideoLatentPipelineDriver, +) from modular_diffusion_nodes_library.latent_pipeline_drivers.ltx import LTXLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.ltx2 import LTX2PipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.qwen import QwenLatentPipelineDriver @@ -18,9 +24,6 @@ from modular_diffusion_nodes_library.latent_pipeline_drivers.stable_diffusion_xl import ( StableDiffusionXLLatentPipelineDriver, ) -from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5 import HunyuanVideo15TextToVideoLatentPipelineDriver -from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5_i2v import HunyuanVideo15ImageToVideoLatentPipelineDriver - from modular_diffusion_nodes_library.latent_pipeline_drivers.wan import WanTextToVideoLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.wan_animate import WanAnimateLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.wan_i2v import WanImageToVideoLatentPipelineDriver diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py index 95dfca1..a54a932 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5.py @@ -1,10 +1,7 @@ import logging -from typing import Any, ClassVar, override +from typing import Any, ClassVar, cast, override import torch # type: ignore[reportMissingImports] -from diffusers.modular_pipelines.hunyuan_video1_5 import ( # type: ignore[reportMissingImports] - HunyuanVideo15AutoBlocks, -) from diffusers.modular_pipelines.hunyuan_video1_5.before_denoise import ( # type: ignore[reportMissingImports] HunyuanVideo15PrepareLatentsStep, HunyuanVideo15SetTimestepsStep, @@ -12,7 +9,21 @@ from diffusers.modular_pipelines.hunyuan_video1_5.decoders import ( # type: ignore[reportMissingImports] HunyuanVideo15VaeDecoderStep, ) -from diffusers.modular_pipelines.modular_pipeline import ModularPipeline # type: ignore[reportMissingImports] +from diffusers.modular_pipelines.hunyuan_video1_5.modular_blocks_hunyuan_video1_5 import ( + HunyuanVideo15AutoBlocks, +) +from diffusers.modular_pipelines.hunyuan_video1_5.modular_pipeline import ( # type: ignore[reportMissingImports] + HunyuanVideo15ModularPipeline, +) +from diffusers.modular_pipelines.modular_pipeline import ( # type: ignore[reportMissingImports] + ModularPipeline, + ModularPipelineBlocks, + PipelineState, +) +from diffusers.modular_pipelines.modular_pipeline_utils import ( # type: ignore[reportMissingImports] + InputParam, + OutputParam, +) from diffusers.pipelines.pipeline_utils import DiffusionPipeline # type: ignore[reportMissingImports] from modular_diffusion_nodes_library.artifact_utils.inpaint_mask_artifact import InpaintMaskArtifact @@ -28,6 +39,94 @@ logger = logging.getLogger("modular_diffusers_nodes_library") +class _HunyuanVideo15EncodeVideoStep(ModularPipelineBlocks): + """Encode a list of PIL frames into a normalised HunyuanVideo 1.5 video latent [B, C, T_lat, H/vsf, W/vsf].""" + + model_name = "hunyuan_video1_5" + + @property + def inputs(self) -> list[InputParam]: + return [InputParam("frames", required=True), InputParam("generator")] + + @property + def intermediate_outputs(self) -> list[OutputParam]: + return [ + OutputParam( + "video_latents", + type_hint=torch.Tensor, + description="Normalised video latent in the HunyuanVideo 1.5 VAE's output space.", + ), + ] + + @torch.no_grad() + def __call__( + self, components: HunyuanVideo15ModularPipeline, state: PipelineState + ) -> tuple[HunyuanVideo15ModularPipeline, PipelineState]: + block_state = cast(Any, self.get_block_state(state)) + + device = components._execution_device + dtype = components.vae.dtype + + frame_tensors = [components.video_processor.preprocess(frame) for frame in block_state.frames] + video_tensor = torch.stack([t.squeeze(0) for t in frame_tensors], dim=0) # [T, C, H, W] + video_tensor = video_tensor.permute(1, 0, 2, 3).unsqueeze(0).to(device=device, dtype=dtype) # [1, C, T, H, W] + + latents = components.vae.encode(video_tensor).latent_dist.sample(generator=block_state.generator) + block_state.video_latents = latents * components.vae.config.scaling_factor + + self.set_block_state(state, block_state) + return components, state + + +class _HunyuanVideo15AddNoiseStep(ModularPipelineBlocks): + """Add flow-matching noise to a HunyuanVideo 1.5 video latent at a strength-derived timestep.""" + + model_name = "hunyuan_video1_5" + + @property + def inputs(self) -> list[InputParam]: + return [ + InputParam("latents", required=True), + InputParam("noise", required=True), + InputParam("num_inference_steps", required=True), + InputParam("strength", required=True), + ] + + @property + def intermediate_outputs(self) -> list[OutputParam]: + return [ + OutputParam( + "noisy_latents", + type_hint=torch.Tensor, + description="Latents with noise added at the strength-derived timestep.", + ), + ] + + @torch.no_grad() + def __call__( + self, components: HunyuanVideo15ModularPipeline, state: PipelineState + ) -> tuple[HunyuanVideo15ModularPipeline, PipelineState]: + block_state = cast(Any, self.get_block_state(state)) + + _, state = HunyuanVideo15SetTimestepsStep()(components, state) # type: ignore[reportOperatorIssue] + timesteps = state.values.get("timesteps") + if timesteps is None or len(timesteps) == 0: + raise ValueError("HunyuanVideo15SetTimestepsStep did not return valid timesteps.") + + num_inference_steps = block_state.num_inference_steps + strength = block_state.strength + init_timestep = min(num_inference_steps * strength, num_inference_steps) + t_start = int(max(num_inference_steps - init_timestep, 0)) + latent_timestep = timesteps[t_start * components.scheduler.order :][:1] + + block_state.noisy_latents = components.scheduler.scale_noise( + block_state.latents, latent_timestep, block_state.noise + ) + + self.set_block_state(state, block_state) + return components, state + + class HunyuanVideo15TextToVideoLatentPipelineDriver(LatentPipelineDriver): produces_video: ClassVar[bool] = True # EXAMPLE_DOC_STRING in pipeline_hunyuan_video1_5.py specifies fps=15 @@ -64,7 +163,7 @@ def create_noise_latent(self, source_shape: tuple[int, ...], generator_state: Ge generator=generator, batch_size=1, ) - latents = output_state.get("latents") + latents = self._get_required(output_state, "latents", torch.Tensor) return self._make_latent_artifact( latents, source_shape=source_shape, @@ -78,30 +177,21 @@ def decode_latent(self, latent: LatentArtifact) -> DecodeResult: latents = latent.to_torch(device=device, dtype=dtype) vae_decoder_step = HunyuanVideo15VaeDecoderStep() output_state = self._call_block(vae_decoder_step, latents=latents, output_type="pil") - video_frames = output_state.get("videos")[0] + video_frames = self._get_required(output_state, "videos", list)[0] return video_frames @override def encode_media(self, media: ImageMedia | VideoMedia, generator_state: GeneratorState) -> LatentArtifact: if isinstance(media, ImageMedia): raise NotImplementedError( - f"Pipeline '{self.pipe.__class__.__name__}' does not support image encoding. " - "Use a video input instead." + f"Pipeline '{self.pipe.__class__.__name__}' does not support image encoding. Use a video input instead." ) - device, dtype = self._get_device_and_type() - vae = self.pipe.vae - frame_tensors = [ - self.pipe.video_processor.preprocess(frame) - for frame in media.frames - ] - video_tensor = torch.stack([t.squeeze(0) for t in frame_tensors], dim=0) # [T, C, H, W] - video_tensor = video_tensor.permute(1, 0, 2, 3).unsqueeze(0) # [1, C, T, H, W] - video_tensor = video_tensor.to(device=device, dtype=vae.dtype) generator = generator_state.to_generator() - with torch.no_grad(): - latents = vae.encode(video_tensor).latent_dist.sample(generator=generator) - latents = latents.to(dtype=dtype) * vae.config.scaling_factor - return self._make_latent_artifact(latents, source_shape=media.source_shape) + output = self._call_block(_HunyuanVideo15EncodeVideoStep(), frames=media.frames, generator=generator) + return self._make_latent_artifact( + self._get_required(output, "video_latents", torch.Tensor), + source_shape=media.source_shape, + ) @override def add_noise_to_latent( @@ -114,18 +204,17 @@ def add_noise_to_latent( device, dtype = self._get_device_and_type() source_shape = latent.source_shape latents = latent.to_torch(device=device, dtype=dtype) - timesteps_state = self._call_block(HunyuanVideo15SetTimestepsStep(), num_inference_steps=num_inference_steps) - timesteps = timesteps_state.get("timesteps") - if timesteps is None or len(timesteps) == 0: - raise ValueError("HunyuanVideo15SetTimestepsStep did not return valid timesteps.") noise_artifact = self.create_noise_latent(source_shape, generator_state) noise = noise_artifact.to_torch(device=device, dtype=dtype) noise_generator_state = GeneratorState.from_artifact(noise_artifact) or generator_state - init_timestep = min(num_inference_steps * strength, num_inference_steps) - t_start = int(max(num_inference_steps - init_timestep, 0)) - latent_timestep = timesteps[t_start * self.pipe.scheduler.order :][:1] - with torch.no_grad(): - result = self.pipe.scheduler.scale_noise(latents, latent_timestep, noise) + output = self._call_block( + _HunyuanVideo15AddNoiseStep(), + latents=latents, + noise=noise, + num_inference_steps=num_inference_steps, + strength=strength, + ) + result = self._get_required(output, "noisy_latents", torch.Tensor) return self._make_latent_artifact( result, source_shape=source_shape, diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py index e8f065c..ecd0ca1 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/hunyuan_video1_5_i2v.py @@ -2,10 +2,10 @@ from typing import Any, ClassVar, override import torch # type: ignore[reportMissingImports] -from PIL.Image import Image, Resampling from diffusers.modular_pipelines.hunyuan_video1_5.encoders import ( # type: ignore[reportMissingImports] HunyuanVideo15VaeEncoderStep, ) +from PIL.Image import Image, Resampling from modular_diffusion_nodes_library.artifact_utils.inpaint_mask_artifact import InpaintMaskArtifact from modular_diffusion_nodes_library.artifact_utils.latent_artifact import LatentArtifact @@ -15,7 +15,9 @@ ImageMedia, VideoMedia, ) -from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5 import HunyuanVideo15TextToVideoLatentPipelineDriver +from modular_diffusion_nodes_library.latent_pipeline_drivers.hunyuan_video1_5 import ( + HunyuanVideo15TextToVideoLatentPipelineDriver, +) from modular_diffusion_nodes_library.parameters.media_gen_conditioning.conditioning_payload import normalize_to_payloads from modular_diffusion_nodes_library.utils.conditioning_utils import ( ConditioningMode, @@ -25,6 +27,8 @@ ) logger = logging.getLogger("modular_diffusers_nodes_library") + + class HunyuanVideo15ImageToVideoLatentPipelineDriver(HunyuanVideo15TextToVideoLatentPipelineDriver): # EXAMPLE_DOC_STRING in pipeline_hunyuan_video1_5_image2video.py specifies fps=24 video_fps: ClassVar[int] = 24 @@ -127,11 +131,12 @@ def denoise_latent( **update_kwargs, ) - def get_resize_dimensions(self, width: int, height: int) -> tuple[int, int]: """Calculate the resize dimensions for a given width and height.""" target_size = self.pipe.transformer.config.target_size if getattr(self.pipe, "transformer", None) else 640 - height, width = self.pipe.video_processor.calculate_default_height_width(height=height, width=width, target_size=target_size) + height, width = self.pipe.video_processor.calculate_default_height_width( + height=height, width=width, target_size=target_size + ) return width, height def preprocess_image(self, image: Image, width: int, height: int) -> Image: @@ -139,4 +144,4 @@ def preprocess_image(self, image: Image, width: int, height: int) -> Image: # Automatically resize image based on model capabilities resized_width, resized_height = self.get_resize_dimensions(width, height) image = image.resize((resized_width, resized_height)) - return image \ No newline at end of file + return image diff --git a/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py b/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py index 7e5d2ac..671ac88 100644 --- a/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py +++ b/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py @@ -222,7 +222,7 @@ def pipeline_type_badge_message(self) -> str: "- `HunyuanVideo15Pipeline` — Text-to-video generation (Tencent HunyuanVideo 1.5).\n" "- `HunyuanVideo15ImageToVideoPipeline` — Image-to-video. Requires a dedicated I2V checkpoint. \n\n" "For first-frame conditioning, connect a Media Gen Conditioning node to the " - "`additional_parameters` input of Generate Media Latents." + "`conditioning_images` input of Generate Media Latents." ) @classmethod @@ -325,7 +325,7 @@ def pipeline_type_badge_message(self) -> str: "- `WanImageToVideoPipeline` — Image-to-video. Requires a dedicated I2V checkpoint. " "Cannot be used with base WAN weights.\n\n" "For first/last-frame conditioning, connect a Media Gen Conditioning node to the " - "`additional_parameters` input of Generate Media Latents." + "`conditioning_images` input of Generate Media Latents." ) @classmethod diff --git a/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_i2v_runtime_parameters.py b/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_i2v_runtime_parameters.py index fc9def2..c15ada8 100644 --- a/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_i2v_runtime_parameters.py +++ b/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_i2v_runtime_parameters.py @@ -66,14 +66,13 @@ def _add_input_parameters(self) -> None: type="float", tooltip=( "Controls how strongly the model follows the text prompt. " - "Higher values produce videos that more closely match the prompt, usually at the expense of quality. " - "Default 7.5 is sourced from the ClassifierFreeGuidance component default." + "Higher values produce videos that more closely match the prompt, usually at the expense of quality." ), ) guidance_scale_param.set_badge( variant="help", title="Guidance scale", - message="For recommended values, reset the node to restore the model author's defaults.", + message="Controls how closely the output follows your prompt. Higher values = stronger prompt adherence, often at the cost of quality. The model default is 7.5.", ) self._node.add_parameter(guidance_scale_param) diff --git a/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_runtime_parameters.py b/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_runtime_parameters.py index b9271da..0510d3a 100644 --- a/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_runtime_parameters.py +++ b/modular_diffusion_nodes_library/runtime_parameters/hunyuan_video1_5_runtime_parameters.py @@ -38,13 +38,12 @@ def _add_input_parameters(self) -> None: tooltip=( "Controls how strongly the model follows the text prompt. " "Higher values produce videos that more closely match the prompt, usually at the expense of quality. " - "Default 7.5 is sourced from the ClassifierFreeGuidance component default." ), ) guidance_scale_param.set_badge( variant="help", title="Guidance scale", - message="For recommended values, reset the node to restore the model author's defaults.", + message="Controls how closely the output follows your prompt. Higher values = stronger prompt adherence, often at the cost of quality. The model default is 7.5.", ) self._node.add_parameter(guidance_scale_param) diff --git a/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py b/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py index 753111a..30116f7 100644 --- a/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py +++ b/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py @@ -19,18 +19,18 @@ from modular_diffusion_nodes_library.runtime_parameters.flux_fill_runtime_parameters import ( FluxFillPipelineRuntimeParameters, ) -from modular_diffusion_nodes_library.runtime_parameters.hunyuan_video1_5_i2v_runtime_parameters import ( - HunyuanVideo15ImageToVideoPipelineRuntimeParameters, -) -from modular_diffusion_nodes_library.runtime_parameters.hunyuan_video1_5_runtime_parameters import ( - HunyuanVideo15PipelineRuntimeParameters, -) from modular_diffusion_nodes_library.runtime_parameters.flux_kontext_runtime_parameters import ( FluxKontextPipelineRuntimeParameters, ) from modular_diffusion_nodes_library.runtime_parameters.flux_runtime_parameters import ( FluxPipelineRuntimeParameters, ) +from modular_diffusion_nodes_library.runtime_parameters.hunyuan_video1_5_i2v_runtime_parameters import ( + HunyuanVideo15ImageToVideoPipelineRuntimeParameters, +) +from modular_diffusion_nodes_library.runtime_parameters.hunyuan_video1_5_runtime_parameters import ( + HunyuanVideo15PipelineRuntimeParameters, +) from modular_diffusion_nodes_library.runtime_parameters.ltx2_runtime_parameters import ( LTX2PipelineRuntimeParameters, )