From 025dfd4c67e28be26cd53e88047b87d944146a94 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Sun, 8 Feb 2026 03:38:45 +0000 Subject: [PATCH 01/19] update create pipeline section --- .../en/modular_diffusers/modular_pipeline.md | 204 +++++++++++------- 1 file changed, 130 insertions(+), 74 deletions(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index 34cd8f72b5b7..c7e0a54bc71f 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -12,27 +12,28 @@ specific language governing permissions and limitations under the License. # ModularPipeline -[`ModularPipeline`] converts [`~modular_pipelines.ModularPipelineBlocks`]'s into an executable pipeline that loads models and performs the computation steps defined in the block. It is the main interface for running a pipeline and it is very similar to the [`DiffusionPipeline`] API. +[`ModularPipeline`] converts [`~modular_pipelines.ModularPipelineBlocks`] into an executable pipeline that loads models and performs the computation steps defined in the blocks. It is the main interface for running a pipeline and the API is very similar to [`DiffusionPipeline`] but with a few key differences. -The main difference is to include an expected `output` argument in the pipeline. +**Loading is lazy.** With [`DiffusionPipeline`], [`~DiffusionPipeline.from_pretrained`] creates the pipeline and loads all models at the same time. With [`ModularPipeline`], creating and loading are two separate steps: [`~ModularPipeline.from_pretrained`] reads the configuration and knows where to load each component from, but doesn't actually load the model weights. You load the models later with [`~ModularPipeline.load_components`], which is where you pass loading arguments like `torch_dtype` and `quantization_config`. + +**Two ways to create a pipeline.** You can use [`~ModularPipeline.from_pretrained`] with an existing diffusers model repository — it automatically maps to the default pipeline blocks and then converts to a [`ModularPipeline`] with no extra setup. Currently supported models include SDXL, Wan, Qwen, Z-Image, Flux, and Flux2. You can also assemble your own pipeline from [`ModularPipelineBlocks`] and convert it with the [`~ModularPipelineBlocks.init_pipeline`] method (see [Creating a pipeline](#creating-a-pipeline) for more details). + +**Running the pipeline is the same.** Once loaded, you call the pipeline with the same arguments you're used to. A single [`ModularPipeline`] can support multiple workflows (text-to-image, image-to-image, inpainting, etc.) when the pipeline blocks use [`AutoPipelineBlocks`](./auto_pipeline) to automatically select the workflow based on your inputs. + +Below are complete examples for text-to-image, image-to-image, and inpainting with SDXL. ```py import torch -from diffusers.modular_pipelines import SequentialPipelineBlocks -from diffusers.modular_pipelines.stable_diffusion_xl import TEXT2IMAGE_BLOCKS - -blocks = SequentialPipelineBlocks.from_blocks_dict(TEXT2IMAGE_BLOCKS) - -modular_repo_id = "YiYiXu/modular-loader-t2i-0704" -pipeline = blocks.init_pipeline(modular_repo_id) +from diffusers import ModularPipeline +pipeline = ModularPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") pipeline.load_components(torch_dtype=torch.float16) pipeline.to("cuda") -image = pipeline(prompt="Astronaut in a jungle, cold color palette, muted colors, detailed, 8k", output="images")[0] +image = pipeline(prompt="Astronaut in a jungle, cold color palette, muted colors, detailed, 8k").images[0] image.save("modular_t2i_out.png") ``` @@ -41,21 +42,17 @@ image.save("modular_t2i_out.png") ```py import torch -from diffusers.modular_pipelines import SequentialPipelineBlocks -from diffusers.modular_pipelines.stable_diffusion_xl import IMAGE2IMAGE_BLOCKS - -blocks = SequentialPipelineBlocks.from_blocks_dict(IMAGE2IMAGE_BLOCKS) - -modular_repo_id = "YiYiXu/modular-loader-t2i-0704" -pipeline = blocks.init_pipeline(modular_repo_id) +from diffusers import ModularPipeline +from diffusers.utils import load_image +pipeline = ModularPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") pipeline.load_components(torch_dtype=torch.float16) pipeline.to("cuda") url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png" init_image = load_image(url) prompt = "a dog catching a frisbee in the jungle" -image = pipeline(prompt=prompt, image=init_image, strength=0.8, output="images")[0] +image = pipeline(prompt=prompt, image=init_image, strength=0.8).images[0] image.save("modular_i2i_out.png") ``` @@ -64,15 +61,10 @@ image.save("modular_i2i_out.png") ```py import torch -from diffusers.modular_pipelines import SequentialPipelineBlocks -from diffusers.modular_pipelines.stable_diffusion_xl import INPAINT_BLOCKS +from diffusers import ModularPipeline from diffusers.utils import load_image -blocks = SequentialPipelineBlocks.from_blocks_dict(INPAINT_BLOCKS) - -modular_repo_id = "YiYiXu/modular-loader-t2i-0704" -pipeline = blocks.init_pipeline(modular_repo_id) - +pipeline = ModularPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") pipeline.load_components(torch_dtype=torch.float16) pipeline.to("cuda") @@ -83,96 +75,160 @@ init_image = load_image(img_url) mask_image = load_image(mask_url) prompt = "A deep sea diver floating" -image = pipeline(prompt=prompt, image=init_image, mask_image=mask_image, strength=0.85, output="images")[0] +image = pipeline(prompt=prompt, image=init_image, mask_image=mask_image, strength=0.85).images[0] image.save("moduar_inpaint_out.png") ``` -This guide will show you how to create a [`ModularPipeline`] and manage the components in it. +This guide will show you how to create a [`ModularPipeline`] and manage the components in it, and run it. -## Adding blocks +## Creating a pipeline -Blocks are [`InsertableDict`] objects that can be inserted at specific positions, providing a flexible way to mix-and-match blocks. +There are two ways to create a [`ModularPipeline`]. Assemble and create a pipeline from [`ModularPipelineBlocks`] with [`~ModularPipelineBlocks.init_pipeline`], or load an existing pipeline with [`~ModularPipeline.from_pretrained`]. -Use [`~modular_pipelines.modular_pipeline_utils.InsertableDict.insert`] on either the block class or `sub_blocks` attribute to add a block. +You should also initialize a [`ComponentsManager`] to handle device placement and memory and component management. +> [!TIP] +> Refer to the [ComponentsManager](./components_manager) doc for more details about how it can help manage components across different workflows. + +### init_pipeline + +[`~ModularPipelineBlocks.init_pipeline`] converts any [`ModularPipelineBlocks`] into a [`ModularPipeline`]. + +Let's define a minimal block to see how it works: ```py -# BLOCKS is dict of block classes, you need to add class to it -BLOCKS.insert("block_name", BlockClass, index) -# sub_blocks attribute contains instance, add a block instance to the attribute -t2i_blocks.sub_blocks.insert("block_name", block_instance, index) -``` +from transformers import CLIPTextModel +from diffusers.modular_pipelines import ( + ComponentSpec, + ModularPipelineBlocks, + PipelineState, +) -Use [`~modular_pipelines.modular_pipeline_utils.InsertableDict.pop`] on either the block class or `sub_blocks` attribute to remove a block. +class MyBlock(ModularPipelineBlocks): + @property + def expected_components(self): + return [ + ComponentSpec( + name="text_encoder", + type_hint=CLIPTextModel, + pretrained_model_name_or_path="openai/clip-vit-large-patch14", + ), + ] + + def __call__(self, components, state: PipelineState) -> PipelineState: + return components, state +``` +Call [`~ModularPipelineBlocks.init_pipeline`] to convert it into a pipeline. The `blocks` attribute on the pipeline is the blocks it was created from — it determines the expected inputs, outputs, and computation logic. ```py -# remove a block class from preset -BLOCKS.pop("text_encoder") -# split out a block instance on its own -text_encoder_block = t2i_blocks.sub_blocks.pop("text_encoder") +block = MyBlock() +pipe = block.init_pipeline() +pipe.blocks +``` +``` +MyBlock { + "_class_name": "MyBlock", + "_diffusers_version": "0.37.0.dev0" +} ``` -Swap blocks by setting the existing block to the new block. +Call [`~ModularPipelineBlocks.init_pipeline`] to convert it into a pipeline. The `blocks` attribute on the pipeline is the blocks it was created from — it determines the expected inputs, outputs, and computation logic. + +> [!WARNING] +> Blocks are mutable — you can freely add, remove, or swap blocks before creating a pipeline. However, once a pipeline is created, modifying `pipeline.blocks` won't affect the pipeline because it returns a copy. If you want a different block structure, create a new pipeline after modifying the blocks. + +When you call [`~ModularPipelineBlocks.init_pipeline`] without a repository, it uses the `pretrained_model_name_or_path` defined in the block's [`ComponentSpec`] to determine where to load each component from. Printing the pipeline shows the component loading configuration. ```py -# Replace block class in preset -BLOCKS["prepare_latents"] = CustomPrepareLatents -# Replace in sub_blocks attribute using an block instance -t2i_blocks.sub_blocks["prepare_latents"] = CustomPrepareLatents() +pipe +ModularPipeline { + "_blocks_class_name": "MyBlock", + "_class_name": "ModularPipeline", + "_diffusers_version": "0.37.0.dev0", + "text_encoder": [ + null, + null, + { + "pretrained_model_name_or_path": "openai/clip-vit-large-patch14", + "revision": null, + "subfolder": "", + "type_hint": [ + "transformers", + "CLIPTextModel" + ], + "variant": null + } + ] +} ``` -## Creating a pipeline +If you pass a repository to [`~ModularPipelineBlocks.init_pipeline`], it overrides the loading path by matching your block's components against the pipeline config in that repository (`model_index.json` or `modular_model_index.json`). -There are two ways to create a [`ModularPipeline`]. Assemble and create a pipeline from [`ModularPipelineBlocks`] or load an existing pipeline with [`~ModularPipeline.from_pretrained`]. +In the example below, the `pretrained_model_name_or_path` will be updated to `"stabilityai/stable-diffusion-xl-base-1.0"`. +```py +pipe = block.init_pipeline("stabilityai/stable-diffusion-xl-base-1.0") +pipe +ModularPipeline { + "_blocks_class_name": "MyBlock", + "_class_name": "ModularPipeline", + "_diffusers_version": "0.37.0.dev0", + "text_encoder": [ + null, + null, + { + "pretrained_model_name_or_path": "stabilityai/stable-diffusion-xl-base-1.0", + "revision": null, + "subfolder": "text_encoder", + "type_hint": [ + "transformers", + "CLIPTextModel" + ], + "variant": null + } + ] +} +``` -You should also initialize a [`ComponentsManager`] to handle device placement and memory and component management. +If a component in your block doesn't exist in the repository, it remains `null` and is skipped during [`~ModularPipeline.load_components`]. -> [!TIP] -> Refer to the [ComponentsManager](./components_manager) doc for more details about how it can help manage components across different workflows. +### from_pretrained - - +[`~ModularPipeline.from_pretrained`] is a convenient way to create a [`ModularPipeline`] without defining blocks yourself. -Use the [`~ModularPipelineBlocks.init_pipeline`] method to create a [`ModularPipeline`] from the component and configuration specifications. This method loads the *specifications* from a `modular_model_index.json` file, but it doesn't load the *models* yet. +It works with three types of repositories. +**A regular diffusers repository.** Pass any supported model repository and it automatically maps to the default pipeline blocks. Currently supported models include SDXL, Wan, Qwen, Z-Image, Flux, and Flux2. ```py -from diffusers import ComponentsManager -from diffusers.modular_pipelines import SequentialPipelineBlocks -from diffusers.modular_pipelines.stable_diffusion_xl import TEXT2IMAGE_BLOCKS - -t2i_blocks = SequentialPipelineBlocks.from_blocks_dict(TEXT2IMAGE_BLOCKS) +from diffusers import ModularPipeline, ComponentsManager -modular_repo_id = "YiYiXu/modular-loader-t2i-0704" components = ComponentsManager() -t2i_pipeline = t2i_blocks.init_pipeline(modular_repo_id, components_manager=components) +pipeline = ModularPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", components_manager=components +) ``` - - - -The [`~ModularPipeline.from_pretrained`] method creates a [`ModularPipeline`] from a modular repository on the Hub. - +**A modular repository.** These repositories contain a `modular_model_index.json` that specifies where to load each component from — the components can come from different repositories and the modular repository itself may not contain any model weights. For example, [diffusers/flux2-bnb-4bit-modular](https://huggingface.co/diffusers/flux2-bnb-4bit-modular) loads a quantized transformer from one repository and the remaining components from another. See [Modular repository](#modular-repository) for more details on the format. ```py from diffusers import ModularPipeline, ComponentsManager components = ComponentsManager() -pipeline = ModularPipeline.from_pretrained("YiYiXu/modular-loader-t2i-0704", components_manager=components) +pipeline = ModularPipeline.from_pretrained( + "diffusers/flux2-bnb-4bit-modular", components_manager=components +) ``` -Add the `trust_remote_code` argument to load a custom [`ModularPipeline`]. - +**A modular repository with custom code.** Some repositories include custom pipeline blocks alongside the loading configuration. Add `trust_remote_code=True` to load them. See [Custom blocks](./custom_blocks) for how to create your own. ```py from diffusers import ModularPipeline, ComponentsManager components = ComponentsManager() -modular_repo_id = "YiYiXu/modular-diffdiff-0704" -diffdiff_pipeline = ModularPipeline.from_pretrained(modular_repo_id, trust_remote_code=True, components_manager=components) +pipeline = ModularPipeline.from_pretrained( + "diffusers/Florence2-image-Annotator", trust_remote_code=True, components_manager=components +) ``` - - ## Loading components @@ -184,7 +240,7 @@ A [`ModularPipeline`] doesn't automatically instantiate with components. It only ```py import torch -t2i_pipeline.load_components(torch_dtype=torch.float16) +pipeline.load_components(torch_dtype=torch.float16) t2i_pipeline.to("cuda") ``` @@ -196,7 +252,7 @@ The example below only loads the UNet and VAE. ```py import torch -t2i_pipeline.load_components(names=["unet", "vae"], torch_dtype=torch.float16) +pipeline.load_components(names=["unet", "vae"], torch_dtype=torch.float16) ``` From 24cbb354c0863338181e1b2514d7a09fc579da9a Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Sun, 8 Feb 2026 08:46:41 +0000 Subject: [PATCH 02/19] update more --- .../en/modular_diffusers/modular_pipeline.md | 181 +++++++++--------- 1 file changed, 86 insertions(+), 95 deletions(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index c7e0a54bc71f..cbe8006abbee 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -232,183 +232,174 @@ pipeline = ModularPipeline.from_pretrained( ## Loading components -A [`ModularPipeline`] doesn't automatically instantiate with components. It only loads the configuration and component specifications. You can load all components with [`~ModularPipeline.load_components`] or only load specific components with [`~ModularPipeline.load_components`]. - - - +A [`ModularPipeline`] doesn't automatically instantiate with components. It only loads the configuration and component specifications. You can load components with [`~ModularPipeline.load_components`]. +This will load all the components that have a valid loading spec. ```py import torch pipeline.load_components(torch_dtype=torch.float16) -t2i_pipeline.to("cuda") ``` - - - -The example below only loads the UNet and VAE. - +You can also load specific components by name. The example below only loads the text_encoder. ```py -import torch - -pipeline.load_components(names=["unet", "vae"], torch_dtype=torch.float16) +pipeline.load_components(names=["text_encoder"], torch_dtype=torch.float16) ``` - - - -Print the pipeline to inspect the loaded pretrained components. - +After loading, printing the pipeline shows which components are loaded — the first two fields change from `null` to the component's library and class. ```py -t2i_pipeline +pipeline ``` - -This should match the `modular_model_index.json` file from the modular repository a pipeline is initialized from. If a pipeline doesn't need a component, it won't be included even if it exists in the modular repository. - -To modify where components are loaded from, edit the `modular_model_index.json` file in the repository and change it to your desired loading path. The example below loads a UNet from a different repository. - -```json -# original -"unet": [ - null, null, - { - "repo": "stabilityai/stable-diffusion-xl-base-1.0", - "subfolder": "unet", - "variant": "fp16" - } +``` +# text_encoder is loaded - shows library and class +"text_encoder": [ + "transformers", + "CLIPTextModel", + { ... } ] -# modified +# unet is not loaded yet - still null "unet": [ - null, null, - { - "repo": "RunDiffusion/Juggernaut-XL-v9", - "subfolder": "unet", - "variant": "fp16" - } + null, + null, + { ... } ] ``` + +Loading keyword arguments like `torch_dtype`, `variant`, `revision`, and `quantization_config` are passed through to `from_pretrained()` for each component. You can pass a single value to apply to all components, or a dict to set per-component values. +```py +# apply bfloat16 to all components +pipeline.load_components(torch_dtype=torch.bfloat16) + +# different dtypes per component +pipeline.load_components(torch_dtype={"transformer": torch.bfloat16, "default": torch.float32}) +``` + +Note that [`~ModularPipeline.load_components`] only loads components that haven't been loaded yet and have a valid loading spec. This means if you've already set a component on the pipeline, calling [`~ModularPipeline.load_components`] again won't reload it. + ### Component loading status The pipeline properties below provide more information about which components are loaded. Use `component_names` to return all expected components. - ```py -t2i_pipeline.component_names +pipeline.component_names ['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'guider', 'scheduler', 'unet', 'vae', 'image_processor'] ``` -Use `null_component_names` to return components that aren't loaded yet. Load these components with [`~ModularPipeline.from_pretrained`]. - +Use `null_component_names` to return components that aren't loaded yet. ```py -t2i_pipeline.null_component_names +pipeline.null_component_names ['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'scheduler'] ``` Use `pretrained_component_names` to return components that will be loaded from pretrained models. - ```py -t2i_pipeline.pretrained_component_names +pipeline.pretrained_component_names ['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'scheduler', 'unet', 'vae'] ``` Use `config_component_names` to return components that are created with the default config (not loaded from a modular repository). Components from a config aren't included because they are already initialized during pipeline creation. This is why they aren't listed in `null_component_names`. ```py -t2i_pipeline.config_component_names +pipeline.config_component_names ['guider', 'image_processor'] ``` ## Updating components -Components may be updated depending on whether it is a *pretrained component* or a *config component*. - -> [!WARNING] -> A component may change from pretrained to config when updating a component. The component type is initially defined in a block's `expected_components` field. - -A pretrained component is updated with [`ComponentSpec`] whereas a config component is updated by eihter passing the object directly or with [`ComponentSpec`]. +[`~ModularPipeline.update_components`] replaces a component on the pipeline with a new one. When a component is updated, the loading specifications are also updated in the pipeline config and [`~ModularPipeline.load_components`] will skip it unless it was expliclty listed in the `names` argument. -The [`ComponentSpec`] shows `default_creation_method="from_pretrained"` for a pretrained component shows `default_creation_method="from_config` for a config component. +There are several ways to load a component to update. -To update a pretrained component, create a [`ComponentSpec`] with the name of the component and where to load it from. Use the [`~ComponentSpec.load`] method to load the component. +### From AutoModel +You can pass a model object loaded with `AutoModel.from_pretrained()`. Models loaded this way are automatically tagged with their loading information. ```py -from diffusers import ComponentSpec, UNet2DConditionModel +from diffusers import AutoModel -unet_spec = ComponentSpec(name="unet",type_hint=UNet2DConditionModel, repo="stabilityai/stable-diffusion-xl-base-1.0", subfolder="unet", variant="fp16") -unet = unet_spec.load(torch_dtype=torch.float16) +unet = AutoModel.from_pretrained( + "RunDiffusion/Juggernaut-XL-v9", subfolder="unet", variant="fp16", torch_dtype=torch.float16 +) +pipeline.update_components(unet=unet) ``` -The [`~ModularPipeline.update_components`] method replaces the component with a new one. +### From ComponentSpec +Use [`~ModularPipeline.get_component_spec`] to get a copy of the current component specification, modify it, and load a new component. ```py -t2i_pipeline.update_components(unet=unet2) -``` +unet_spec = pipeline.get_component_spec("unet") -When a component is updated, the loading specifications are also updated in the pipeline config. +# modify to load from a different repository +unet_spec.pretrained_model_name_or_path = "RunDiffusion/Juggernaut-XL-v9" + +# load and update +unet = unet_spec.load(torch_dtype=torch.float16) +pipeline.update_components(unet=unet) +``` -### Component extraction and modification +You can also create a [`ComponentSpec`] from scratch. -When you use [`~ComponentSpec.load`], the new component maintains its loading specifications. This makes it possible to extract the specification and recreate the component. +Not all components are loaded from pretrained weights — some are created from a config (listed under `pipeline.config_component_names`). For these, use [`~ComponentSpec.create`] instead of [`~ComponentSpec.load`]. ```py -spec = ComponentSpec.from_component("unet", unet2) -spec -ComponentSpec(name='unet', type_hint=, description=None, config=None, repo='stabilityai/stable-diffusion-xl-base-1.0', subfolder='unet', variant='fp16', revision=None, default_creation_method='from_pretrained') -unet2_recreated = spec.load(torch_dtype=torch.float16) +guider_spec = pipeline.get_component_spec("guider") +guider_spec.config = {"guidance_scale": 5.0} +guider = guider_spec.create() +pipeline.update_components(guider=guider) ``` -The [`~ModularPipeline.get_component_spec`] method gets a copy of the current component specification to modify or update. - +Or simply pass the object directly. ```py -unet_spec = t2i_pipeline.get_component_spec("unet") -unet_spec -ComponentSpec( - name='unet', - type_hint=, - pretrained_model_name_or_path='RunDiffusion/Juggernaut-XL-v9', - subfolder='unet', - variant='fp16', - default_creation_method='from_pretrained' -) +from diffusers.guiders import ClassifierFreeGuidance -# modify to load from a different repository -unet_spec.pretrained_model_name_or_path = "stabilityai/stable-diffusion-xl-base-1.0" - -# load component with modified spec -unet = unet_spec.load(torch_dtype=torch.float16) +guider = ClassifierFreeGuidance(guidance_scale=5.0) +pipeline.update_components(guider=guider) ``` +See the [Guiders](./guiders) guide for more details on available guiders and how to configure them. + ## Modular repository A repository is required if the pipeline blocks use *pretrained components*. The repository supplies loading specifications and metadata. -[`ModularPipeline`] specifically requires *modular repositories* (see [example repository](https://huggingface.co/YiYiXu/modular-diffdiff)) which are more flexible than a typical repository. It contains a `modular_model_index.json` file containing the following 3 elements. +[`ModularPipeline`] works with regular diffusers repositories out of the box. However, you can also create a *modular repository* for more flexibility. A modular repository contains a `modular_model_index.json` file containing the following 3 elements. -- `library` and `class` shows which library the component was loaded from and it's class. If `null`, the component hasn't been loaded yet. +- `library` and `class` shows which library the component was loaded from and its class. If `null`, the component hasn't been loaded yet. - `loading_specs_dict` contains the information required to load the component such as the repository and subfolder it is loaded from. -Unlike standard repositories, a modular repository can fetch components from different repositories based on the `loading_specs_dict`. Components don't need to exist in the same repository. +The key advantage of a modular repository is that components can be loaded from different repositories. For example, [diffusers/flux2-bnb-4bit-modular](https://huggingface.co/diffusers/flux2-bnb-4bit-modular) loads a quantized transformer from `diffusers/FLUX.2-dev-bnb-4bit` while loading the remaining components from `black-forest-labs/FLUX.2-dev`. -A modular repository may contain custom code for loading a [`ModularPipeline`]. This allows you to use specialized blocks that aren't native to Diffusers. +To convert a regular diffusers repository into a modular one, create the pipeline using the regular repository, and then save it using `save_pretrained()`. The saved repository will contain a `modular_model_index.json` with all the loading specifications. Optionnally, you can pass a repo_id and push_to_hub=True to publich the modular repo on Huggingface Hub. + +```py +from diffusers import ModularPipeline +# load from a regular repo +pipeline = ModularPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") + +# push as a modular repository +pipeline.save_pretrained("local/path", repo_id = ..., push_to_hub=True) +``` + +A modular repository can also include custom pipeline blocks as Python code. This allows you to share specialized blocks that aren't native to Diffusers. For example, [diffusers/Florence2-image-Annotator](https://huggingface.co/diffusers/Florence2-image-Annotator) contains custom blocks alongside the loading configuration: ``` -modular-diffdiff-0704/ +Florence2-image-Annotator/ ├── block.py # Custom pipeline blocks implementation ├── config.json # Pipeline configuration and auto_map +├── mellon_config.json # UI configuration for Mellon └── modular_model_index.json # Component loading specifications ``` -The [config.json](https://huggingface.co/YiYiXu/modular-diffdiff-0704/blob/main/config.json) file contains an `auto_map` key that points to where a custom block is defined in `block.py`. - +The `config.json` file contains an `auto_map` key that tells [`ModularPipeline`] where to find the custom blocks: ```json { - "_class_name": "DiffDiffBlocks", + "_class_name": "Florence2AnnotatorBlocks", "auto_map": { - "ModularPipelineBlocks": "block.DiffDiffBlocks" + "ModularPipelineBlocks": "block.Florence2AnnotatorBlocks" } } ``` + +Load custom code repositories with `trust_remote_code=True` as shown in [from_pretrained](#from_pretrained). See [Custom blocks](./custom_blocks) for how to create and share your own. From 7fdddf012ee6cb883b73f3bf0c0d99c138ac4ebc Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Sun, 8 Feb 2026 08:52:06 +0000 Subject: [PATCH 03/19] update more --- .../en/modular_diffusers/modular_pipeline.md | 66 +++++++------------ 1 file changed, 24 insertions(+), 42 deletions(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index cbe8006abbee..dbccbd3e1ec8 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -76,13 +76,13 @@ mask_image = load_image(mask_url) prompt = "A deep sea diver floating" image = pipeline(prompt=prompt, image=init_image, mask_image=mask_image, strength=0.85).images[0] -image.save("moduar_inpaint_out.png") +image.save("modular_inpaint_out.png") ``` -This guide will show you how to create a [`ModularPipeline`] and manage the components in it, and run it. +This guide will show you how to create a [`ModularPipeline`], manage the components in it, and run it. ## Creating a pipeline @@ -98,6 +98,7 @@ You should also initialize a [`ComponentsManager`] to handle device placement an [`~ModularPipelineBlocks.init_pipeline`] converts any [`ModularPipelineBlocks`] into a [`ModularPipeline`]. Let's define a minimal block to see how it works: + ```py from transformers import CLIPTextModel from diffusers.modular_pipelines import ( @@ -122,11 +123,13 @@ class MyBlock(ModularPipelineBlocks): ``` Call [`~ModularPipelineBlocks.init_pipeline`] to convert it into a pipeline. The `blocks` attribute on the pipeline is the blocks it was created from — it determines the expected inputs, outputs, and computation logic. + ```py block = MyBlock() pipe = block.init_pipeline() pipe.blocks ``` + ``` MyBlock { "_class_name": "MyBlock", @@ -134,8 +137,6 @@ MyBlock { } ``` -Call [`~ModularPipelineBlocks.init_pipeline`] to convert it into a pipeline. The `blocks` attribute on the pipeline is the blocks it was created from — it determines the expected inputs, outputs, and computation logic. - > [!WARNING] > Blocks are mutable — you can freely add, remove, or swap blocks before creating a pipeline. However, once a pipeline is created, modifying `pipeline.blocks` won't affect the pipeline because it returns a copy. If you want a different block structure, create a new pipeline after modifying the blocks. @@ -167,6 +168,7 @@ ModularPipeline { If you pass a repository to [`~ModularPipelineBlocks.init_pipeline`], it overrides the loading path by matching your block's components against the pipeline config in that repository (`model_index.json` or `modular_model_index.json`). In the example below, the `pretrained_model_name_or_path` will be updated to `"stabilityai/stable-diffusion-xl-base-1.0"`. + ```py pipe = block.init_pipeline("stabilityai/stable-diffusion-xl-base-1.0") pipe @@ -200,6 +202,7 @@ If a component in your block doesn't exist in the repository, it remains `null` It works with three types of repositories. **A regular diffusers repository.** Pass any supported model repository and it automatically maps to the default pipeline blocks. Currently supported models include SDXL, Wan, Qwen, Z-Image, Flux, and Flux2. + ```py from diffusers import ModularPipeline, ComponentsManager @@ -210,6 +213,7 @@ pipeline = ModularPipeline.from_pretrained( ``` **A modular repository.** These repositories contain a `modular_model_index.json` that specifies where to load each component from — the components can come from different repositories and the modular repository itself may not contain any model weights. For example, [diffusers/flux2-bnb-4bit-modular](https://huggingface.co/diffusers/flux2-bnb-4bit-modular) loads a quantized transformer from one repository and the remaining components from another. See [Modular repository](#modular-repository) for more details on the format. + ```py from diffusers import ModularPipeline, ComponentsManager @@ -220,6 +224,7 @@ pipeline = ModularPipeline.from_pretrained( ``` **A modular repository with custom code.** Some repositories include custom pipeline blocks alongside the loading configuration. Add `trust_remote_code=True` to load them. See [Custom blocks](./custom_blocks) for how to create your own. + ```py from diffusers import ModularPipeline, ComponentsManager @@ -229,12 +234,12 @@ pipeline = ModularPipeline.from_pretrained( ) ``` - ## Loading components A [`ModularPipeline`] doesn't automatically instantiate with components. It only loads the configuration and component specifications. You can load components with [`~ModularPipeline.load_components`]. This will load all the components that have a valid loading spec. + ```py import torch @@ -242,14 +247,17 @@ pipeline.load_components(torch_dtype=torch.float16) ``` You can also load specific components by name. The example below only loads the text_encoder. + ```py pipeline.load_components(names=["text_encoder"], torch_dtype=torch.float16) ``` After loading, printing the pipeline shows which components are loaded — the first two fields change from `null` to the component's library and class. + ```py pipeline ``` + ``` # text_encoder is loaded - shows library and class "text_encoder": [ @@ -266,8 +274,8 @@ pipeline ] ``` - Loading keyword arguments like `torch_dtype`, `variant`, `revision`, and `quantization_config` are passed through to `from_pretrained()` for each component. You can pass a single value to apply to all components, or a dict to set per-component values. + ```py # apply bfloat16 to all components pipeline.load_components(torch_dtype=torch.bfloat16) @@ -278,44 +286,14 @@ pipeline.load_components(torch_dtype={"transformer": torch.bfloat16, "default": Note that [`~ModularPipeline.load_components`] only loads components that haven't been loaded yet and have a valid loading spec. This means if you've already set a component on the pipeline, calling [`~ModularPipeline.load_components`] again won't reload it. -### Component loading status - -The pipeline properties below provide more information about which components are loaded. - -Use `component_names` to return all expected components. -```py -pipeline.component_names -['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'guider', 'scheduler', 'unet', 'vae', 'image_processor'] -``` - -Use `null_component_names` to return components that aren't loaded yet. -```py -pipeline.null_component_names -['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'scheduler'] -``` - -Use `pretrained_component_names` to return components that will be loaded from pretrained models. -```py -pipeline.pretrained_component_names -['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'scheduler', 'unet', 'vae'] -``` - -Use `config_component_names` to return components that are created with the default config (not loaded from a modular repository). Components from a config aren't included because they are already initialized during pipeline creation. This is why they aren't listed in `null_component_names`. - -```py -pipeline.config_component_names -['guider', 'image_processor'] -``` - ## Updating components -[`~ModularPipeline.update_components`] replaces a component on the pipeline with a new one. When a component is updated, the loading specifications are also updated in the pipeline config and [`~ModularPipeline.load_components`] will skip it unless it was expliclty listed in the `names` argument. - -There are several ways to load a component to update. +[`~ModularPipeline.update_components`] replaces a component on the pipeline with a new one. When a component is updated, the loading specifications are also updated in the pipeline config and [`~ModularPipeline.load_components`] will skip it on subsequent calls. ### From AutoModel You can pass a model object loaded with `AutoModel.from_pretrained()`. Models loaded this way are automatically tagged with their loading information. + ```py from diffusers import AutoModel @@ -328,6 +306,7 @@ pipeline.update_components(unet=unet) ### From ComponentSpec Use [`~ModularPipeline.get_component_spec`] to get a copy of the current component specification, modify it, and load a new component. + ```py unet_spec = pipeline.get_component_spec("unet") @@ -341,8 +320,8 @@ pipeline.update_components(unet=unet) You can also create a [`ComponentSpec`] from scratch. - Not all components are loaded from pretrained weights — some are created from a config (listed under `pipeline.config_component_names`). For these, use [`~ComponentSpec.create`] instead of [`~ComponentSpec.load`]. + ```py guider_spec = pipeline.get_component_spec("guider") guider_spec.config = {"guidance_scale": 5.0} @@ -351,6 +330,7 @@ pipeline.update_components(guider=guider) ``` Or simply pass the object directly. + ```py from diffusers.guiders import ClassifierFreeGuidance @@ -371,7 +351,7 @@ A repository is required if the pipeline blocks use *pretrained components*. The The key advantage of a modular repository is that components can be loaded from different repositories. For example, [diffusers/flux2-bnb-4bit-modular](https://huggingface.co/diffusers/flux2-bnb-4bit-modular) loads a quantized transformer from `diffusers/FLUX.2-dev-bnb-4bit` while loading the remaining components from `black-forest-labs/FLUX.2-dev`. -To convert a regular diffusers repository into a modular one, create the pipeline using the regular repository, and then save it using `save_pretrained()`. The saved repository will contain a `modular_model_index.json` with all the loading specifications. Optionnally, you can pass a repo_id and push_to_hub=True to publich the modular repo on Huggingface Hub. +To convert a regular diffusers repository into a modular one, create the pipeline using the regular repository, and then save it using `save_pretrained()`. The saved repository will contain a `modular_model_index.json` with all the loading specifications. Optionnally, you can pass a `repo_id` and `push_to_hub=True` to publish it on Huggingface Hub. ```py from diffusers import ModularPipeline @@ -380,10 +360,11 @@ from diffusers import ModularPipeline pipeline = ModularPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") # push as a modular repository -pipeline.save_pretrained("local/path", repo_id = ..., push_to_hub=True) +pipeline.save_pretrained("local/path", repo_id="my-username/sdxl-modular", push_to_hub=True) ``` A modular repository can also include custom pipeline blocks as Python code. This allows you to share specialized blocks that aren't native to Diffusers. For example, [diffusers/Florence2-image-Annotator](https://huggingface.co/diffusers/Florence2-image-Annotator) contains custom blocks alongside the loading configuration: + ``` Florence2-image-Annotator/ ├── block.py # Custom pipeline blocks implementation @@ -393,6 +374,7 @@ Florence2-image-Annotator/ ``` The `config.json` file contains an `auto_map` key that tells [`ModularPipeline`] where to find the custom blocks: + ```json { "_class_name": "Florence2AnnotatorBlocks", @@ -402,4 +384,4 @@ The `config.json` file contains an `auto_map` key that tells [`ModularPipeline`] } ``` -Load custom code repositories with `trust_remote_code=True` as shown in [from_pretrained](#from_pretrained). See [Custom blocks](./custom_blocks) for how to create and share your own. +Load custom code repositories with `trust_remote_code=True` as shown in [from_pretrained](#from_pretrained). See [Custom blocks](./custom_blocks) for how to create and share your own. \ No newline at end of file From 64a90fc2e2050d2aca378723bb8d7c2d83a568c1 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Sun, 8 Feb 2026 09:22:16 +0000 Subject: [PATCH 04/19] more --- docs/source/en/modular_diffusers/modular_pipeline.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index dbccbd3e1ec8..b6b140116b13 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -363,17 +363,16 @@ pipeline = ModularPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base pipeline.save_pretrained("local/path", repo_id="my-username/sdxl-modular", push_to_hub=True) ``` -A modular repository can also include custom pipeline blocks as Python code. This allows you to share specialized blocks that aren't native to Diffusers. For example, [diffusers/Florence2-image-Annotator](https://huggingface.co/diffusers/Florence2-image-Annotator) contains custom blocks alongside the loading configuration: +A modular repository may contain custom code for loading a [`ModularPipeline`]. This allows you to use specialized blocks that aren’t native to Diffusers. For example, [diffusers/Florence2-image-Annotator](https://huggingface.co/diffusers/Florence2-image-Annotator) contains custom blocks alongside the loading configuration: ``` Florence2-image-Annotator/ ├── block.py # Custom pipeline blocks implementation -├── config.json # Pipeline configuration and auto_map -├── mellon_config.json # UI configuration for Mellon +├── modular_config.json # Pipeline configuration and auto_map └── modular_model_index.json # Component loading specifications ``` -The `config.json` file contains an `auto_map` key that tells [`ModularPipeline`] where to find the custom blocks: +The `modular_config.json` file contains an `auto_map` key that tells [`ModularPipeline`] where to find the custom blocks: ```json { From 98ea6e0b2e971b4d00ea5e60ed8777358eaf2b58 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Sun, 8 Feb 2026 09:31:38 +0000 Subject: [PATCH 05/19] add a section on running pipeline moduarly --- .../en/modular_diffusers/modular_pipeline.md | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index b6b140116b13..312633fe2fe8 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -340,6 +340,46 @@ pipeline.update_components(guider=guider) See the [Guiders](./guiders) guide for more details on available guiders and how to configure them. +## Splitting a pipeline into stages + +Since blocks are composable, you can take a pipeline apart and reconstruct it into separate pipelines for each stage. The example below shows how we can separate the text encoder block from the rest of the pipeline, so you can encode the prompt independently and pass the embeddings to the main pipeline. + +```py +from diffusers import ModularPipeline, ComponentsManager +import torch + +device = "cuda" +dtype = torch.bfloat16 +repo_id = "black-forest-labs/FLUX.2-klein-4B" + +# get the blocks and separate out the text encoder +blocks = ModularPipeline.from_pretrained(repo_id).blocks +text_block = blocks.sub_blocks.pop("text_encoder") + +# use ComponentsManager to handle offloading across multiple pipelines +manager = ComponentsManager() +manager.enable_auto_cpu_offload(device=device) + +# create separate pipelines for each stage +text_encoder_pipeline = text_block.init_pipeline(repo_id, components_manager=manager) +pipeline = blocks.init_pipeline(repo_id, components_manager=manager) + +# encode text +text_encoder_pipeline.load_components(torch_dtype=dtype) +text_embeddings = text_encoder_pipeline(prompt="a cat").get_by_kwargs("denoiser_input_fields") + +# denoise and decode +pipeline.load_components(torch_dtype=dtype) +output = pipeline( + **text_embeddings, + num_inference_steps=4, +).images[0] +``` + +[`ComponentsManager`] handles memory across multiple pipelines. Unlike the offloading strategies in [`DiffusionPipeline`] that follow a fixed order, [`ComponentsManager`] makes offloading decisions dynamically each time a model forward pass runs, based on the current memory situation. This means it works regardless of how many pipelines you create or what order you run them in. See the [ComponentsManager](./components_manager) guide for more details. + +If pipeline stages share components (e.g., the same VAE used for encoding and decoding), you can use [`~ModularPipeline.update_components`] to pass an already-loaded component to another pipeline instead of loading it again. + ## Modular repository A repository is required if the pipeline blocks use *pretrained components*. The repository supplies loading specifications and metadata. @@ -351,7 +391,7 @@ A repository is required if the pipeline blocks use *pretrained components*. The The key advantage of a modular repository is that components can be loaded from different repositories. For example, [diffusers/flux2-bnb-4bit-modular](https://huggingface.co/diffusers/flux2-bnb-4bit-modular) loads a quantized transformer from `diffusers/FLUX.2-dev-bnb-4bit` while loading the remaining components from `black-forest-labs/FLUX.2-dev`. -To convert a regular diffusers repository into a modular one, create the pipeline using the regular repository, and then save it using `save_pretrained()`. The saved repository will contain a `modular_model_index.json` with all the loading specifications. Optionnally, you can pass a `repo_id` and `push_to_hub=True` to publish it on Huggingface Hub. +To convert a regular diffusers repository into a modular one, create the pipeline using the regular repository, and then push to the Hub. The saved repository will contain a `modular_model_index.json` with all the loading specifications. ```py from diffusers import ModularPipeline @@ -363,16 +403,17 @@ pipeline = ModularPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base pipeline.save_pretrained("local/path", repo_id="my-username/sdxl-modular", push_to_hub=True) ``` -A modular repository may contain custom code for loading a [`ModularPipeline`]. This allows you to use specialized blocks that aren’t native to Diffusers. For example, [diffusers/Florence2-image-Annotator](https://huggingface.co/diffusers/Florence2-image-Annotator) contains custom blocks alongside the loading configuration: +A modular repository can also include custom pipeline blocks as Python code. This allows you to share specialized blocks that aren't native to Diffusers. For example, [diffusers/Florence2-image-Annotator](https://huggingface.co/diffusers/Florence2-image-Annotator) contains custom blocks alongside the loading configuration: ``` Florence2-image-Annotator/ ├── block.py # Custom pipeline blocks implementation -├── modular_config.json # Pipeline configuration and auto_map +├── config.json # Pipeline configuration and auto_map +├── mellon_config.json # UI configuration for Mellon └── modular_model_index.json # Component loading specifications ``` -The `modular_config.json` file contains an `auto_map` key that tells [`ModularPipeline`] where to find the custom blocks: +The `config.json` file contains an `auto_map` key that tells [`ModularPipeline`] where to find the custom blocks: ```json { From 64dba68e0a83ffd5137f68c00b3d5aa8680a75b1 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Sun, 8 Feb 2026 09:31:55 +0000 Subject: [PATCH 06/19] refactor update_components, remove support for spec --- .../modular_pipelines/modular_pipeline.py | 69 ++++--------------- 1 file changed, 12 insertions(+), 57 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index a5695736581f..83e12e8dd7c4 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -2016,57 +2016,31 @@ def update_components(self, **kwargs): - the `config` dict, which will be saved as `modular_model_index.json` during `save_pretrained` Args: - **kwargs: Component objects, ComponentSpec objects, or configuration values to update: - - Component objects: Only supports components we can extract specs using - `ComponentSpec.from_component()` method i.e. components created with ComponentSpec.load() or - ConfigMixin subclasses that aren't nn.Modules (e.g., `unet=new_unet, text_encoder=new_encoder`) - - ComponentSpec objects: Only supports default_creation_method == "from_config", will call create() - method to create a new component (e.g., `guider=ComponentSpec(name="guider", - type_hint=ClassifierFreeGuidance, config={...}, default_creation_method="from_config")`) - - Configuration values: Simple values to update configuration settings (e.g., - `requires_safety_checker=False`) - - Raises: - ValueError: If a component object is not supported in ComponentSpec.from_component() method: - - nn.Module components without a valid `_diffusers_load_id` attribute - - Non-ConfigMixin components without a valid `_diffusers_load_id` attribute + **kwargs: Component objects or configuration values to update: + - Component objects: Models loaded with `AutoModel.from_pretrained()` or `ComponentSpec.load()` + are automatically tagged with loading information. ConfigMixin objects without weights + (e.g., schedulers, guiders) can be passed directly. + - Configuration values: Simple values to update configuration settings + (e.g., `requires_safety_checker=False`) Examples: ```python - # Update multiple components at once + # Update pretrrained model pipeline.update_components(unet=new_unet_model, text_encoder=new_text_encoder) # Update configuration values pipeline.update_components(requires_safety_checker=False) - # Update both components and configs together - pipeline.update_components(unet=new_unet_model, requires_safety_checker=False) - - # Update with ComponentSpec objects (from_config only) - pipeline.update_components( - guider=ComponentSpec( - name="guider", - type_hint=ClassifierFreeGuidance, - config={"guidance_scale": 5.0}, - default_creation_method="from_config", - ) - ) ``` Notes: - - Components with trained weights must be created using ComponentSpec.load(). If the component has not been - shared in huggingface hub and you don't have loading specs, you can upload it using `push_to_hub()` - - ConfigMixin objects without weights (e.g., schedulers, guiders) can be passed directly - - ComponentSpec objects with default_creation_method="from_pretrained" are not supported in - update_components() + - Components with trained weights should be loaded with `AutoModel.from_pretrained()` or + `ComponentSpec.load()` so that loading specs are preserved for serialization. + - ConfigMixin objects without weights (e.g., schedulers, guiders) can be passed directly. """ - # extract component_specs_updates & config_specs_updates from `specs` - passed_component_specs = { - k: kwargs.pop(k) for k in self._component_specs if k in kwargs and isinstance(kwargs[k], ComponentSpec) - } passed_components = { - k: kwargs.pop(k) for k in self._component_specs if k in kwargs and not isinstance(kwargs[k], ComponentSpec) + k: kwargs.pop(k) for k in self._component_specs if k in kwargs } passed_config_values = {k: kwargs.pop(k) for k in self._config_specs if k in kwargs} @@ -2106,33 +2080,14 @@ def update_components(self, **kwargs): if len(kwargs) > 0: logger.warning(f"Unexpected keyword arguments, will be ignored: {kwargs.keys()}") - created_components = {} - for name, component_spec in passed_component_specs.items(): - if component_spec.default_creation_method == "from_pretrained": - raise ValueError( - "ComponentSpec object with default_creation_method == 'from_pretrained' is not supported in update_components() method" - ) - created_components[name] = component_spec.create() - current_component_spec = self._component_specs[name] - # warn if type changed - if current_component_spec.type_hint is not None and not isinstance( - created_components[name], current_component_spec.type_hint - ): - logger.info( - f"ModularPipeline.update_components: adding {name} with new type: {created_components[name].__class__.__name__}, previous type: {current_component_spec.type_hint.__name__}" - ) - # update _component_specs based on the user passed component_spec - self._component_specs[name] = component_spec - self.register_components(**passed_components, **created_components) + self.register_components(**passed_components) config_to_register = {} for name, new_value in passed_config_values.items(): - # e.g. requires_aesthetics_score = False self._config_specs[name].default = new_value config_to_register[name] = new_value self.register_to_config(**config_to_register) - # YiYi TODO: support map for additional from_pretrained kwargs def load_components(self, names: Optional[Union[List[str], str]] = None, **kwargs): """ Load selected components from specs. From 7224beb0362bc15f19c552cdf2f675d1dd930444 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Sun, 8 Feb 2026 09:34:06 +0000 Subject: [PATCH 07/19] style --- src/diffusers/modular_pipelines/modular_pipeline.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 83e12e8dd7c4..c0ee79cac471 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -2018,8 +2018,8 @@ def update_components(self, **kwargs): Args: **kwargs: Component objects or configuration values to update: - Component objects: Models loaded with `AutoModel.from_pretrained()` or `ComponentSpec.load()` - are automatically tagged with loading information. ConfigMixin objects without weights - (e.g., schedulers, guiders) can be passed directly. + are automatically tagged with loading information. ConfigMixin objects without weights (e.g., + schedulers, guiders) can be passed directly. - Configuration values: Simple values to update configuration settings (e.g., `requires_safety_checker=False`) @@ -2030,7 +2030,6 @@ def update_components(self, **kwargs): # Update configuration values pipeline.update_components(requires_safety_checker=False) - ``` Notes: @@ -2039,9 +2038,7 @@ def update_components(self, **kwargs): - ConfigMixin objects without weights (e.g., schedulers, guiders) can be passed directly. """ - passed_components = { - k: kwargs.pop(k) for k in self._component_specs if k in kwargs - } + passed_components = {k: kwargs.pop(k) for k in self._component_specs if k in kwargs} passed_config_values = {k: kwargs.pop(k) for k in self._config_specs if k in kwargs} for name, component in passed_components.items(): From 86fc6691cbfd5a0ff978d7576a379864b28255d9 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Sun, 8 Feb 2026 09:37:28 +0000 Subject: [PATCH 08/19] bullet points --- docs/source/en/modular_diffusers/modular_pipeline.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index 312633fe2fe8..14d1ac0baa01 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -14,11 +14,11 @@ specific language governing permissions and limitations under the License. [`ModularPipeline`] converts [`~modular_pipelines.ModularPipelineBlocks`] into an executable pipeline that loads models and performs the computation steps defined in the blocks. It is the main interface for running a pipeline and the API is very similar to [`DiffusionPipeline`] but with a few key differences. -**Loading is lazy.** With [`DiffusionPipeline`], [`~DiffusionPipeline.from_pretrained`] creates the pipeline and loads all models at the same time. With [`ModularPipeline`], creating and loading are two separate steps: [`~ModularPipeline.from_pretrained`] reads the configuration and knows where to load each component from, but doesn't actually load the model weights. You load the models later with [`~ModularPipeline.load_components`], which is where you pass loading arguments like `torch_dtype` and `quantization_config`. +- **Loading is lazy.** With [`DiffusionPipeline`], [`~DiffusionPipeline.from_pretrained`] creates the pipeline and loads all models at the same time. With [`ModularPipeline`], creating and loading are two separate steps: [`~ModularPipeline.from_pretrained`] reads the configuration and knows where to load each component from, but doesn't actually load the model weights. You load the models later with [`~ModularPipeline.load_components`], which is where you pass loading arguments like `torch_dtype` and `quantization_config`. -**Two ways to create a pipeline.** You can use [`~ModularPipeline.from_pretrained`] with an existing diffusers model repository — it automatically maps to the default pipeline blocks and then converts to a [`ModularPipeline`] with no extra setup. Currently supported models include SDXL, Wan, Qwen, Z-Image, Flux, and Flux2. You can also assemble your own pipeline from [`ModularPipelineBlocks`] and convert it with the [`~ModularPipelineBlocks.init_pipeline`] method (see [Creating a pipeline](#creating-a-pipeline) for more details). +- **Two ways to create a pipeline.** You can use [`~ModularPipeline.from_pretrained`] with an existing diffusers model repository — it automatically maps to the default pipeline blocks and then converts to a [`ModularPipeline`] with no extra setup. Currently supported models include SDXL, Wan, Qwen, Z-Image, Flux, and Flux2. You can also assemble your own pipeline from [`ModularPipelineBlocks`] and convert it with the [`~ModularPipelineBlocks.init_pipeline`] method (see [Creating a pipeline](#creating-a-pipeline) for more details). -**Running the pipeline is the same.** Once loaded, you call the pipeline with the same arguments you're used to. A single [`ModularPipeline`] can support multiple workflows (text-to-image, image-to-image, inpainting, etc.) when the pipeline blocks use [`AutoPipelineBlocks`](./auto_pipeline) to automatically select the workflow based on your inputs. +- **Running the pipeline is the same.** Once loaded, you call the pipeline with the same arguments you're used to. A single [`ModularPipeline`] can support multiple workflows (text-to-image, image-to-image, inpainting, etc.) when the pipeline blocks use [`AutoPipelineBlocks`](./auto_pipeline) to automatically select the workflow based on your inputs. Below are complete examples for text-to-image, image-to-image, and inpainting with SDXL. From 143f27dcaefdffe5ee77f10871f23ab4b340a88b Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Mon, 9 Feb 2026 02:12:02 +0000 Subject: [PATCH 09/19] update the pipeline block --- .../en/modular_diffusers/pipeline_block.md | 162 +++++++++++++----- 1 file changed, 116 insertions(+), 46 deletions(-) diff --git a/docs/source/en/modular_diffusers/pipeline_block.md b/docs/source/en/modular_diffusers/pipeline_block.md index 06c115e1fb52..274d1096b19b 100644 --- a/docs/source/en/modular_diffusers/pipeline_block.md +++ b/docs/source/en/modular_diffusers/pipeline_block.md @@ -25,81 +25,151 @@ This guide will show you how to create a [`~modular_pipelines.ModularPipelineBlo A [`~modular_pipelines.ModularPipelineBlocks`] requires `inputs`, and `intermediate_outputs`. -- `inputs` are values provided by a user and retrieved from the [`~modular_pipelines.PipelineState`]. This is useful because some workflows resize an image, but the original image is still required. The [`~modular_pipelines.PipelineState`] maintains the original image. +- `inputs` are values a block reads from the [`~modular_pipelines.PipelineState`] to perform its computation. These can be values provided by a user (like a prompt or image) or values produced by a previous block (like encoded image_latents). Use `InputParam` to define `inputs`. - ```py - from diffusers.modular_pipelines import InputParam - - user_inputs = [ - InputParam(name="image", type_hint="PIL.Image", description="raw input image to process") - ] - ``` +```py +class ImageEncodeStep(ModularPipelineBlocks): + ... + + @property + def inputs(self): + return [ + InputParam(name="image", type_hint="PIL.Image", required=True, description="raw input image to process"), + ] + ... +``` -- `intermediate_outputs` are new values created by a block and added to the [`~modular_pipelines.PipelineState`]. The `intermediate_outputs` are available as `inputs` for subsequent blocks or available as the final output from running the pipeline. +-- `intermediate_outputs` are new values created by a block and added to the [`~modular_pipelines.PipelineState`]. The `intermediate_outputs` are available as `inputs` for subsequent blocks or available as the final output from running the pipeline. Use `OutputParam` to define `intermediate_outputs`. - ```py - from diffusers.modular_pipelines import OutputParam +```py +class ImageEncodeStep(ModularPipelineBlocks): + ... - user_intermediate_outputs = [ - OutputParam(name="image_latents", description="latents representing the image") - ] - ``` + @property + def intermediate_outputs(self): + return [ + OutputParam(name="image_latents", description="latents representing the image"), + ] + + ... +``` The intermediate inputs and outputs share data to connect blocks. They are accessible at any point, allowing you to track the workflow's progress. +## Components and configs + +The components and pipeline-level configs a block needs are specified in [`ComponentSpec`] and [`~modular_pipelines.ConfigSpec`]. + +- [`ComponentSpec`] contains the expected components used by a block. You need the `name` of the component and ideally a `type_hint` that specifies exactly what the component is. +- [`~modular_pipelines.ConfigSpec`] contains pipeline-level settings that control behavior across all blocks. + +```py +class ImageEncodeStep(ModularPipelineBlocks): + ... + + @property + def expected_components(self): + return [ + ComponentSpec(name="vae", type_hint=AutoencoderKL), + ] + + @property + def expected_configs(self): + return [ + ConfigSpec("force_zeros_for_empty_prompt", True), + ] + + ... +``` + +When the blocks are converted into a pipeline, the components become available to the block as the first argument in `__call__`. + ## Computation logic The computation a block performs is defined in the `__call__` method and it follows a specific structure. -1. Retrieve the [`~modular_pipelines.BlockState`] to get a local view of the `inputs` +1. Retrieve the [`~modular_pipelines.BlockState`] to get a local view of the `inputs`. 2. Implement the computation logic on the `inputs`. 3. Update [`~modular_pipelines.PipelineState`] to push changes from the local [`~modular_pipelines.BlockState`] back to the global [`~modular_pipelines.PipelineState`]. 4. Return the components and state which becomes available to the next block. ```py -def __call__(self, components, state): - # Get a local view of the state variables this block needs - block_state = self.get_block_state(state) +class ImageEncodeStep(ModularPipelineBlocks): + + def __call__(self, components, state): + # Get a local view of the state variables this block needs + block_state = self.get_block_state(state) - # Your computation logic here - # block_state contains all your inputs - # Access them like: block_state.image, block_state.processed_image + # Your computation logic here + # block_state contains all your inputs + # Access them like: block_state.image, block_state.processed_image - # Update the pipeline state with your updated block_states - self.set_block_state(state, block_state) - return components, state + # Update the pipeline state with your updated block_states + self.set_block_state(state, block_state) + return components, state ``` -### Components and configs +## Putting it all together -The components and pipeline-level configs a block needs are specified in [`ComponentSpec`] and [`~modular_pipelines.ConfigSpec`]. +Here is the complete block with all the pieces connected. -- [`ComponentSpec`] contains the expected components used by a block. You need the `name` of the component and ideally a `type_hint` that specifies exactly what the component is. -- [`~modular_pipelines.ConfigSpec`] contains pipeline-level settings that control behavior across all blocks. +```py +from diffusers import ComponentSpec, AutoencoderKL +from diffusers.modular_pipelines import InputParam, ModularPipelineBlocks, OutputParam + + +class ImageEncodeStep(ModularPipelineBlocks): + + @property + def description(self): + return "Encode an image into latent space." + + @property + def expected_components(self): + return [ + ComponentSpec(name="vae", type_hint=AutoencoderKL), + ] + + @property + def inputs(self): + return [ + InputParam(name="image", type_hint="PIL.Image", required=True, description="raw input image to process"), + ] + + @property + def intermediate_outputs(self): + return [ + OutputParam(name="image_latents", type_hint="torch.Tensor", description="latents representing the image"), + ] + + def __call__(self, components, state): + block_state = self.get_block_state(state) + block_state.image_latents = components.vae.encode(block_state.image) + self.set_block_state(state, block_state) + return components, state +``` + +Every block has a `doc` property that is automatically generated from the properties you defined above. It provides a summary of the block's description, components, inputs, and outputs. ```py -from diffusers import ComponentSpec, ConfigSpec +block = ImageEncoderStep() +print(block.doc) +class ImageEncodeStep -expected_components = [ - ComponentSpec(name="unet", type_hint=UNet2DConditionModel), - ComponentSpec(name="scheduler", type_hint=EulerDiscreteScheduler) -] + Encode an image into latent space. -expected_config = [ - ConfigSpec("force_zeros_for_empty_prompt", True) -] -``` + Components: + vae (`AutoencoderKL`) -When the blocks are converted into a pipeline, the components become available to the block as the first argument in `__call__`. + Inputs: + image (`PIL.Image`): + raw input image to process -```py -def __call__(self, components, state): - # Access components using dot notation - unet = components.unet - vae = components.vae - scheduler = components.scheduler -``` + Outputs: + image_latents (`torch.Tensor`): + latents representing the image +``` \ No newline at end of file From de67dc9d7d886ef4e4b7583d84f46da51a9526d1 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Mon, 9 Feb 2026 02:17:57 +0000 Subject: [PATCH 10/19] small fix in state doc --- docs/source/en/modular_diffusers/modular_diffusers_states.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/en/modular_diffusers/modular_diffusers_states.md b/docs/source/en/modular_diffusers/modular_diffusers_states.md index eb55b524e491..657b088fe485 100644 --- a/docs/source/en/modular_diffusers/modular_diffusers_states.md +++ b/docs/source/en/modular_diffusers/modular_diffusers_states.md @@ -25,9 +25,7 @@ This guide explains how states work and how they connect blocks. The [`~modular_pipelines.PipelineState`] is a global state container for all blocks. It maintains the complete runtime state of the pipeline and provides a structured way for blocks to read from and write to shared data. -There are two dict's in [`~modular_pipelines.PipelineState`] for structuring data. - -- The `values` dict is a **mutable** state containing a copy of user provided input values and intermediate output values generated by blocks. If a block modifies an `input`, it will be reflected in the `values` dict after calling `set_block_state`. +[`~modular_pipelines.PipelineState`] stores all data in a `values` dict, which is a **mutable** state containing user provided input values and intermediate output values generated by blocks. If a block modifies an `input`, it will be reflected in the `values` dict after calling `set_block_state`. ```py PipelineState( From 7f784dfc54a70a28022ca6b037176ea826de3a9b Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Mon, 9 Feb 2026 02:54:47 +0000 Subject: [PATCH 11/19] update sequential doc --- .../sequential_pipeline_blocks.md | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index f1549a26b86f..6a26b98ebdfd 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -91,23 +91,42 @@ class ImageEncoderBlock(ModularPipelineBlocks): -Connect the two blocks by defining an [`InsertableDict`] to map the block names to the block instances. Blocks are executed in the order they're registered in `blocks_dict`. - -Use [`~modular_pipelines.SequentialPipelineBlocks.from_blocks_dict`] to create a [`~modular_pipelines.SequentialPipelineBlocks`]. +Connect the two blocks by defining a [~modular_pipelines.SequentialPipelineBlocks]. List the block instances in `block_classes` and their corresponding names in `block_names`. The blocks are executed in the order they appear in `block_classes`, and data flows from one block to the next through [~modular_pipelines.PipelineState]. ```py -from diffusers.modular_pipelines import SequentialPipelineBlocks, InsertableDict - -blocks_dict = InsertableDict() -blocks_dict["input"] = input_block -blocks_dict["image_encoder"] = image_encoder_block +class ImageProcessingStep(SequentialPipelineBlocks): + """ + # auto_docstring + """ + model_name = "my_model" + block_classes = [InputBlock(), ImageEncoderBlock()] + block_names = ["input", "image_encoder"] -blocks = SequentialPipelineBlocks.from_blocks_dict(blocks_dict) + @property + def description(self): + return ( + "Process text prompts and images for the pipeline. It:\n" + " - Determines the batch size from the prompts.\n" + " - Encodes the image into latent space." + ) ``` -Inspect the sub-blocks in [`~modular_pipelines.SequentialPipelineBlocks`] by calling `blocks`, and for more details about the inputs and outputs, access the `docs` attribute. +When you create a [~modular_pipelines.SequentialPipelineBlocks], properties like `inputs`, `intermediate_outputs`, and `expected_components` are automatically aggregated from the sub-blocks, so there is no need to define them again. + +There are a few properties you should set: + +- `description`: We recommend adding a description for the assembled block to explain what the combined step does. +- `model_name`: This is automatically derived from the sub-blocks but isn't always correct, so you may need to override it. +- `outputs`: By default this is the same as `intermediate_outputs`, but you can manually set it to control which values appear in the doc. This is useful for showing only the final outputs instead of all intermediate values. + +These properties, together with the aggregated `inputs`, `intermediate_outputs`, and `expected_components`, are used to automatically generate the `doc` property. + + +Inspect the sub-blocks through the `sub_blocks` property, and use `doc` for a full summary of the block's inputs, outputs, and components. + ```py +blocks = ImageProcessingStep() print(blocks) print(blocks.doc) -``` +``` \ No newline at end of file From 1d95dd3e1f1bda97edf2bd8413887fe6b62ea388 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Mon, 9 Feb 2026 02:59:13 +0000 Subject: [PATCH 12/19] fix link --- docs/source/en/modular_diffusers/modular_pipeline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index 14d1ac0baa01..cb2d8fa79f64 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -18,7 +18,7 @@ specific language governing permissions and limitations under the License. - **Two ways to create a pipeline.** You can use [`~ModularPipeline.from_pretrained`] with an existing diffusers model repository — it automatically maps to the default pipeline blocks and then converts to a [`ModularPipeline`] with no extra setup. Currently supported models include SDXL, Wan, Qwen, Z-Image, Flux, and Flux2. You can also assemble your own pipeline from [`ModularPipelineBlocks`] and convert it with the [`~ModularPipelineBlocks.init_pipeline`] method (see [Creating a pipeline](#creating-a-pipeline) for more details). -- **Running the pipeline is the same.** Once loaded, you call the pipeline with the same arguments you're used to. A single [`ModularPipeline`] can support multiple workflows (text-to-image, image-to-image, inpainting, etc.) when the pipeline blocks use [`AutoPipelineBlocks`](./auto_pipeline) to automatically select the workflow based on your inputs. +- **Running the pipeline is the same.** Once loaded, you call the pipeline with the same arguments you're used to. A single [`ModularPipeline`] can support multiple workflows (text-to-image, image-to-image, inpainting, etc.) when the pipeline blocks use [`AutoPipelineBlocks`](./auto_pipeline_blocks) to automatically select the workflow based on your inputs. Below are complete examples for text-to-image, image-to-image, and inpainting with SDXL. From 6bf733a7445465abc11edf03b48310272db88109 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Mon, 9 Feb 2026 08:04:22 +0000 Subject: [PATCH 13/19] small update on quikstart --- .../source/en/modular_diffusers/quickstart.md | 67 +++++++++++++++---- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/docs/source/en/modular_diffusers/quickstart.md b/docs/source/en/modular_diffusers/quickstart.md index 5a455f0b3093..77d0a2e9c467 100644 --- a/docs/source/en/modular_diffusers/quickstart.md +++ b/docs/source/en/modular_diffusers/quickstart.md @@ -39,17 +39,44 @@ image [`~ModularPipeline.from_pretrained`] uses lazy loading - it reads the configuration to learn where to load each component from, but doesn't actually load the model weights until you call [`~ModularPipeline.load_components`]. This gives you control over when and how components are loaded. > [!TIP] -> [`ComponentsManager`] with `enable_auto_cpu_offload` automatically moves models between CPU and GPU as needed, reducing memory usage for large models like Qwen-Image. Learn more in the [ComponentsManager](./components_manager) guide. +> `ComponentsManager` with `enable_auto_cpu_offload` automatically moves models between CPU and GPU as needed, reducing memory usage for large models like Qwen-Image. Learn more in the [ComponentsManager](./components_manager) guide. +> +> If you don't need offloading, simply remove the `components_manager` argument and move the pipeline to your device manually with `pipe.to("cuda")`. Learn more about creating and loading pipelines in the [Creating a pipeline](https://huggingface.co/docs/diffusers/modular_diffusers/modular_pipeline#creating-a-pipeline) and [Loading components](https://huggingface.co/docs/diffusers/modular_diffusers/modular_pipeline#loading-components) guides. ## Understand the structure -A [`ModularPipeline`] has two parts: -- **State**: the loaded components (models, schedulers, processors) and configuration -- **Definition**: the [`ModularPipelineBlocks`] that specify inputs, outputs, expected components and computation logic +A [`ModularPipeline`] has two parts: a **definition** (the blocks) and a **state** (the loaded components and configs). -The blocks define *what* the pipeline does. Access them through `pipe.blocks`. +Print the pipeline to see its state — the components and their loading status and configuration. +```py +print(pipe) +``` +``` +QwenImageModularPipeline { + "_blocks_class_name": "QwenImageAutoBlocks", + "_class_name": "QwenImageModularPipeline", + "_diffusers_version": "0.37.0.dev0", + "transformer": [ + "diffusers", + "QwenImageTransformer2DModel", + { + "pretrained_model_name_or_path": "Qwen/Qwen-Image", + "revision": null, + "subfolder": "transformer", + "type_hint": [ + "diffusers", + "QwenImageTransformer2DModel" + ], + "variant": null + } + ], + ... +} +``` + +Access the definition through `pipe.blocks` — this is the [`~modular_pipelines.ModularPipelineBlocks`] that defines the pipeline's workflows, inputs, outputs, and computation logic. ```py print(pipe.blocks) ``` @@ -87,7 +114,9 @@ The output returns: ### Workflows -`QwenImageAutoBlocks` is a [`ConditionalPipelineBlocks`], so this pipeline supports multiple workflows and adapts its behavior based on the inputs you provide. For example, if you pass `image` to the pipeline, it runs an image-to-image workflow instead of text-to-image. Let's see this in action with an example. +This pipeline supports multiple workflows and adapts its behavior based on the inputs you provide. For example, if you pass `image` to the pipeline, it runs an image-to-image workflow instead of text-to-image. Learn more about how this works under the hood in the [AutoPipelineBlocks](https://huggingface.co/docs/diffusers/modular_diffusers/auto_pipeline_blocks) guide. + +Let's see this in action with an example. ```py from diffusers.utils import load_image @@ -99,20 +128,21 @@ image = pipe( ).images[0] ``` -Use `get_workflow()` to extract the blocks for a specific workflow. Pass the workflow name (e.g., `"image2image"`, `"inpainting"`, `"controlnet_text2image"`) to get only the blocks relevant to that workflow. +Use `get_workflow()` to extract the blocks for a specific workflow. Pass the workflow name (e.g., `"image2image"`, `"inpainting"`, `"controlnet_text2image"`) to get only the blocks relevant to that workflow. This is useful when you want to customize or debug a specific workflow. ```py img2img_blocks = pipe.blocks.get_workflow("image2image") ``` -Conditional blocks are convenient for users, but their conditional logic adds complexity when customizing or debugging. Extracting a workflow gives you the specific blocks relevant to your workflow, making it easier to work with. Learn more in the [AutoPipelineBlocks](https://huggingface.co/docs/diffusers/modular_diffusers/auto_pipeline_blocks) guide. ### Sub-blocks Blocks can contain other blocks. `pipe.blocks` gives you the top-level block definition (here, `QwenImageAutoBlocks`), while `sub_blocks` lets you access the smaller blocks inside it. -`QwenImageAutoBlocks` is composed of: `text_encoder`, `vae_encoder`, `controlnet_vae_encoder`, `denoise`, and `decode`. Access them through the `sub_blocks` property. +`QwenImageAutoBlocks` is composed of: `text_encoder`, `vae_encoder`, `controlnet_vae_encoder`, `denoise`, and `decode`. -The `doc` property is useful for seeing the full documentation of any block, including its inputs, outputs, and components. +These sub-blocks run one after another and data flows linearly from one block to the next — each block's `intermediate_outputs` become available as `inputs` to the next block. This is how [`SequentialPipelineBlocks`](./sequential_pipeline_blocks) work. + +You can access them through the `sub_blocks` property. The `doc` property is useful for seeing the full documentation of any block, including its inputs, outputs, and components. ```py vae_encoder_block = pipe.blocks.sub_blocks["vae_encoder"] print(vae_encoder_block.doc) @@ -165,7 +195,7 @@ class CannyBlock Canny map for input image ``` -UUse `get_workflow` to extract the ControlNet workflow from [`QwenImageAutoBlocks`]. +Use `get_workflow` to extract the ControlNet workflow from [`QwenImageAutoBlocks`]. ```py # Get the controlnet workflow that we want to work with blocks = pipe.blocks.get_workflow("controlnet_text2image") @@ -182,9 +212,8 @@ class SequentialPipelineBlocks ... ``` -The extracted workflow is a [`SequentialPipelineBlocks`](./sequential_pipeline_blocks) - a multi-block type where blocks run one after another and data flows linearly from one block to the next. Each block's `intermediate_outputs` become available as `inputs` to subsequent blocks. -Currently this workflow requires `control_image` as input. Let's insert the canny block at the beginning so the pipeline accepts a regular image instead. +The extracted workflow is a [`SequentialPipelineBlocks`](./sequential_pipeline_blocks) and it currently requires `control_image` as input. Let's insert the canny block at the beginning so the pipeline accepts a regular image instead. ```py # Insert canny at the beginning blocks.sub_blocks.insert("canny", canny_block, 0) @@ -211,7 +240,7 @@ class SequentialPipelineBlocks Now the pipeline takes `image` as input instead of `control_image`. Because blocks in a sequence share data automatically, the canny block's output (`control_image`) flows to the denoise block that needs it, and the canny block's input (`image`) becomes a pipeline input since no earlier block provides it. -Create a pipeline from the modified blocks and load a ControlNet model. +Create a pipeline from the modified blocks and load a ControlNet model. The ControlNet isn't part of the original model repository, so we load it separately and add it with [`~ModularPipeline.update_components`]. ```py pipeline = blocks.init_pipeline("Qwen/Qwen-Image", components_manager=manager) @@ -241,6 +270,16 @@ output ## Next steps + + +Understand the core building blocks of Modular Diffusers: + +- [ModularPipelineBlocks](./pipeline_block): The basic unit for defining a step in a pipeline. +- [SequentialPipelineBlocks](./sequential_pipeline_blocks): Chain blocks to run in sequence. +- [AutoPipelineBlocks](./auto_pipeline_blocks): Create pipelines that support multiple workflows. +- [States](./modular_diffusers_states): How data is shared between blocks. + + Learn how to create your own blocks with custom logic in the [Building Custom Blocks](./custom_blocks) guide. From 81eeac7e8d8adfe1b91494f0ba40918f37417928 Mon Sep 17 00:00:00 2001 From: "yiyi@huggingface.co" Date: Mon, 9 Feb 2026 08:09:54 +0000 Subject: [PATCH 14/19] add a note on how to run pipeline without the componen4ts manager --- docs/source/en/modular_diffusers/modular_pipeline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index cb2d8fa79f64..d102d048270a 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -88,7 +88,7 @@ This guide will show you how to create a [`ModularPipeline`], manage the compone There are two ways to create a [`ModularPipeline`]. Assemble and create a pipeline from [`ModularPipelineBlocks`] with [`~ModularPipelineBlocks.init_pipeline`], or load an existing pipeline with [`~ModularPipeline.from_pretrained`]. -You should also initialize a [`ComponentsManager`] to handle device placement and memory and component management. +You can also initialize a [`ComponentsManager`](./components_manager) to handle device placement and memory management. If you don't need automatic offloading, you can skip this and move the pipeline to your device manually with `pipeline.to("cuda")`. > [!TIP] > Refer to the [ComponentsManager](./components_manager) doc for more details about how it can help manage components across different workflows. From ec3b0539824fad8dbe80bbe737b9c53793531410 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sat, 14 Feb 2026 07:11:46 -1000 Subject: [PATCH 15/19] Apply suggestions from code review Co-authored-by: Sayak Paul Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- docs/source/en/modular_diffusers/modular_pipeline.md | 6 +++--- docs/source/en/modular_diffusers/pipeline_block.md | 4 ++-- docs/source/en/modular_diffusers/quickstart.md | 6 +++--- .../en/modular_diffusers/sequential_pipeline_blocks.md | 6 +++--- src/diffusers/modular_pipelines/modular_pipeline.py | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index d102d048270a..2b7f04ad48ca 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -82,7 +82,7 @@ image.save("modular_inpaint_out.png") -This guide will show you how to create a [`ModularPipeline`], manage the components in it, and run it. +This guide will show you how to create a [`ModularPipeline`], manage its components, and run the pipeline. ## Creating a pipeline @@ -246,7 +246,7 @@ import torch pipeline.load_components(torch_dtype=torch.float16) ``` -You can also load specific components by name. The example below only loads the text_encoder. +You can also load specific components by name. The example below only loads the `text_encoder`. ```py pipeline.load_components(names=["text_encoder"], torch_dtype=torch.float16) @@ -284,7 +284,7 @@ pipeline.load_components(torch_dtype=torch.bfloat16) pipeline.load_components(torch_dtype={"transformer": torch.bfloat16, "default": torch.float32}) ``` -Note that [`~ModularPipeline.load_components`] only loads components that haven't been loaded yet and have a valid loading spec. This means if you've already set a component on the pipeline, calling [`~ModularPipeline.load_components`] again won't reload it. +[`~ModularPipeline.load_components`] only loads components that haven't been loaded yet and have a valid loading spec. This means if you've already set a component on the pipeline, calling [`~ModularPipeline.load_components`] again won't reload it. ## Updating components diff --git a/docs/source/en/modular_diffusers/pipeline_block.md b/docs/source/en/modular_diffusers/pipeline_block.md index 274d1096b19b..612736f3d307 100644 --- a/docs/source/en/modular_diffusers/pipeline_block.md +++ b/docs/source/en/modular_diffusers/pipeline_block.md @@ -25,7 +25,7 @@ This guide will show you how to create a [`~modular_pipelines.ModularPipelineBlo A [`~modular_pipelines.ModularPipelineBlocks`] requires `inputs`, and `intermediate_outputs`. -- `inputs` are values a block reads from the [`~modular_pipelines.PipelineState`] to perform its computation. These can be values provided by a user (like a prompt or image) or values produced by a previous block (like encoded image_latents). +- `inputs` are values a block reads from the [`~modular_pipelines.PipelineState`] to perform its computation. These can be values provided by a user (like a prompt or image) or values produced by a previous block (like encoded `image_latents`). Use `InputParam` to define `inputs`. @@ -41,7 +41,7 @@ class ImageEncodeStep(ModularPipelineBlocks): ... ``` --- `intermediate_outputs` are new values created by a block and added to the [`~modular_pipelines.PipelineState`]. The `intermediate_outputs` are available as `inputs` for subsequent blocks or available as the final output from running the pipeline. +- `intermediate_outputs` are new values created by a block and added to the [`~modular_pipelines.PipelineState`]. The `intermediate_outputs` are available as `inputs` for subsequent blocks or available as the final output from running the pipeline. Use `OutputParam` to define `intermediate_outputs`. diff --git a/docs/source/en/modular_diffusers/quickstart.md b/docs/source/en/modular_diffusers/quickstart.md index 77d0a2e9c467..fb7ac1ad9402 100644 --- a/docs/source/en/modular_diffusers/quickstart.md +++ b/docs/source/en/modular_diffusers/quickstart.md @@ -41,7 +41,7 @@ image > [!TIP] > `ComponentsManager` with `enable_auto_cpu_offload` automatically moves models between CPU and GPU as needed, reducing memory usage for large models like Qwen-Image. Learn more in the [ComponentsManager](./components_manager) guide. > -> If you don't need offloading, simply remove the `components_manager` argument and move the pipeline to your device manually with `pipe.to("cuda")`. +> If you don't need offloading, remove the `components_manager` argument and move the pipeline to your device manually with `to("cuda")`. Learn more about creating and loading pipelines in the [Creating a pipeline](https://huggingface.co/docs/diffusers/modular_diffusers/modular_pipeline#creating-a-pipeline) and [Loading components](https://huggingface.co/docs/diffusers/modular_diffusers/modular_pipeline#loading-components) guides. @@ -213,7 +213,7 @@ class SequentialPipelineBlocks ``` -The extracted workflow is a [`SequentialPipelineBlocks`](./sequential_pipeline_blocks) and it currently requires `control_image` as input. Let's insert the canny block at the beginning so the pipeline accepts a regular image instead. +The extracted workflow is a [`SequentialPipelineBlocks`](./sequential_pipeline_blocks) and it currently requires `control_image` as input. Insert the canny block at the beginning so the pipeline accepts a regular image instead. ```py # Insert canny at the beginning blocks.sub_blocks.insert("canny", canny_block, 0) @@ -240,7 +240,7 @@ class SequentialPipelineBlocks Now the pipeline takes `image` as input instead of `control_image`. Because blocks in a sequence share data automatically, the canny block's output (`control_image`) flows to the denoise block that needs it, and the canny block's input (`image`) becomes a pipeline input since no earlier block provides it. -Create a pipeline from the modified blocks and load a ControlNet model. The ControlNet isn't part of the original model repository, so we load it separately and add it with [`~ModularPipeline.update_components`]. +Create a pipeline from the modified blocks and load a ControlNet model. The ControlNet isn't part of the original model repository, so load it separately and add it with [`~ModularPipeline.update_components`]. ```py pipeline = blocks.init_pipeline("Qwen/Qwen-Image", components_manager=manager) diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index 6a26b98ebdfd..1bd67e17b8bf 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -91,7 +91,7 @@ class ImageEncoderBlock(ModularPipelineBlocks): -Connect the two blocks by defining a [~modular_pipelines.SequentialPipelineBlocks]. List the block instances in `block_classes` and their corresponding names in `block_names`. The blocks are executed in the order they appear in `block_classes`, and data flows from one block to the next through [~modular_pipelines.PipelineState]. +Connect the two blocks by defining a [`~modular_pipelines.SequentialPipelineBlocks`]. List the block instances in `block_classes` and their corresponding names in `block_names`. The blocks are executed in the order they appear in `block_classes`, and data flows from one block to the next through [`~modular_pipelines.PipelineState`]. ```py class ImageProcessingStep(SequentialPipelineBlocks): @@ -111,7 +111,7 @@ class ImageProcessingStep(SequentialPipelineBlocks): ) ``` -When you create a [~modular_pipelines.SequentialPipelineBlocks], properties like `inputs`, `intermediate_outputs`, and `expected_components` are automatically aggregated from the sub-blocks, so there is no need to define them again. +When you create a [`~modular_pipelines.SequentialPipelineBlocks`], properties like `inputs`, `intermediate_outputs`, and `expected_components` are automatically aggregated from the sub-blocks, so there is no need to define them again. There are a few properties you should set: @@ -122,7 +122,7 @@ There are a few properties you should set: These properties, together with the aggregated `inputs`, `intermediate_outputs`, and `expected_components`, are used to automatically generate the `doc` property. -Inspect the sub-blocks through the `sub_blocks` property, and use `doc` for a full summary of the block's inputs, outputs, and components. +Print the `ImageProcessingStep` block to inspect its sub-blocks, and use `doc` for a full summary of the block's inputs, outputs, and components. ```py diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index bf3dd9a3521a..efaabd69dc94 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -2076,7 +2076,7 @@ def update_components(self, **kwargs): Examples: ```python - # Update pretrrained model + # Update pre-trained model pipeline.update_components(unet=new_unet_model, text_encoder=new_text_encoder) # Update configuration values From 7ad84ac2394305850ee7fb9888fd57dfe21a1d65 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 14 Feb 2026 19:43:04 +0100 Subject: [PATCH 16/19] remove the supported models mention --- docs/source/en/modular_diffusers/modular_pipeline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/modular_diffusers/modular_pipeline.md b/docs/source/en/modular_diffusers/modular_pipeline.md index 2b7f04ad48ca..e28e13ed5655 100644 --- a/docs/source/en/modular_diffusers/modular_pipeline.md +++ b/docs/source/en/modular_diffusers/modular_pipeline.md @@ -16,7 +16,7 @@ specific language governing permissions and limitations under the License. - **Loading is lazy.** With [`DiffusionPipeline`], [`~DiffusionPipeline.from_pretrained`] creates the pipeline and loads all models at the same time. With [`ModularPipeline`], creating and loading are two separate steps: [`~ModularPipeline.from_pretrained`] reads the configuration and knows where to load each component from, but doesn't actually load the model weights. You load the models later with [`~ModularPipeline.load_components`], which is where you pass loading arguments like `torch_dtype` and `quantization_config`. -- **Two ways to create a pipeline.** You can use [`~ModularPipeline.from_pretrained`] with an existing diffusers model repository — it automatically maps to the default pipeline blocks and then converts to a [`ModularPipeline`] with no extra setup. Currently supported models include SDXL, Wan, Qwen, Z-Image, Flux, and Flux2. You can also assemble your own pipeline from [`ModularPipelineBlocks`] and convert it with the [`~ModularPipelineBlocks.init_pipeline`] method (see [Creating a pipeline](#creating-a-pipeline) for more details). +- **Two ways to create a pipeline.** You can use [`~ModularPipeline.from_pretrained`] with an existing diffusers model repository — it automatically maps to the default pipeline blocks and then converts to a [`ModularPipeline`] with no extra setup. You can check the [modular_pipelines_directory](https://github.com/huggingface/diffusers/tree/main/src/diffusers/modular_pipelines) to see which models are currently supported. You can also assemble your own pipeline from [`ModularPipelineBlocks`] and convert it with the [`~ModularPipelineBlocks.init_pipeline`] method (see [Creating a pipeline](#creating-a-pipeline) for more details). - **Running the pipeline is the same.** Once loaded, you call the pipeline with the same arguments you're used to. A single [`ModularPipeline`] can support multiple workflows (text-to-image, image-to-image, inpainting, etc.) when the pipeline blocks use [`AutoPipelineBlocks`](./auto_pipeline_blocks) to automatically select the workflow based on your inputs. From 3eb031bf7f01cbb60cd81fbde17704b2f1dd19ae Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 14 Feb 2026 19:54:15 +0100 Subject: [PATCH 17/19] update more --- docs/source/en/modular_diffusers/quickstart.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/en/modular_diffusers/quickstart.md b/docs/source/en/modular_diffusers/quickstart.md index fb7ac1ad9402..884495e107b4 100644 --- a/docs/source/en/modular_diffusers/quickstart.md +++ b/docs/source/en/modular_diffusers/quickstart.md @@ -116,7 +116,6 @@ The output returns: This pipeline supports multiple workflows and adapts its behavior based on the inputs you provide. For example, if you pass `image` to the pipeline, it runs an image-to-image workflow instead of text-to-image. Learn more about how this works under the hood in the [AutoPipelineBlocks](https://huggingface.co/docs/diffusers/modular_diffusers/auto_pipeline_blocks) guide. -Let's see this in action with an example. ```py from diffusers.utils import load_image @@ -128,7 +127,7 @@ image = pipe( ).images[0] ``` -Use `get_workflow()` to extract the blocks for a specific workflow. Pass the workflow name (e.g., `"image2image"`, `"inpainting"`, `"controlnet_text2image"`) to get only the blocks relevant to that workflow. This is useful when you want to customize or debug a specific workflow. +Use `get_workflow()` to extract the blocks for a specific workflow. Pass the workflow name (e.g., `"image2image"`, `"inpainting"`, `"controlnet_text2image"`) to get only the blocks relevant to that workflow. This is useful when you want to customize or debug a specific workflow. You can check `pipe.blocks.available_workflows` to see all available workflows. ```py img2img_blocks = pipe.blocks.get_workflow("image2image") ``` From 0e9bc171fa0be15ad705486fa5159ab4e3789440 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 14 Feb 2026 20:01:06 +0100 Subject: [PATCH 18/19] up --- src/diffusers/modular_pipelines/modular_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 69413369aac7..8ae56601eb95 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -19,7 +19,7 @@ from collections import OrderedDict from copy import deepcopy from dataclasses import dataclass, field -from typing import Any +from typing import Any, List, Optional, Union import torch from huggingface_hub import create_repo @@ -403,7 +403,7 @@ def save_pretrained(self, save_directory, push_to_hub=False, **kwargs): def init_pipeline( self, - pretrained_model_name_or_path: str | os.PathLike | None = None, + pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, components_manager: ComponentsManager | None = None, collection: str | None = None, ) -> "ModularPipeline": From 298bf7fbf3739c4ba788fdb48b4d94e40470cccc Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 14 Feb 2026 21:16:33 +0100 Subject: [PATCH 19/19] revert type hint changes --- src/diffusers/modular_pipelines/modular_pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 8ae56601eb95..0eff85926fc5 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -19,7 +19,7 @@ from collections import OrderedDict from copy import deepcopy from dataclasses import dataclass, field -from typing import Any, List, Optional, Union +from typing import Any import torch from huggingface_hub import create_repo @@ -403,7 +403,7 @@ def save_pretrained(self, save_directory, push_to_hub=False, **kwargs): def init_pipeline( self, - pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, + pretrained_model_name_or_path: str | os.PathLike | None = None, components_manager: ComponentsManager | None = None, collection: str | None = None, ) -> "ModularPipeline": @@ -2136,7 +2136,7 @@ def update_components(self, **kwargs): config_to_register[name] = new_value self.register_to_config(**config_to_register) - def load_components(self, names: Optional[Union[List[str], str]] = None, **kwargs): + def load_components(self, names: list[str] | str | None = None, **kwargs): """ Load selected components from specs.