From a6505c222c3b7c5cba33a10cd316c50ac84bf59d Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 18 Mar 2025 09:51:30 +0530 Subject: [PATCH 1/6] deprecate blip. --- src/diffusers/pipelines/__init__.py | 6 +- .../pipelines/controlnet/__init__.py | 2 - .../pipelines/deprecated/__init__.py | 4 ++ .../blip_diffusion/__init__.py | 4 +- .../blip_diffusion/blip_image_processing.py | 0 .../blip_diffusion/modeling_blip2.py | 0 .../blip_diffusion/modeling_ctx_clip.py | 0 .../blip_diffusion/pipeline_blip_diffusion.py | 0 .../deprecated/controlnet/__init__.py | 58 +++++++++++++++++++ .../pipeline_controlnet_blip_diffusion.py | 0 10 files changed, 66 insertions(+), 8 deletions(-) rename src/diffusers/pipelines/{ => deprecated}/blip_diffusion/__init__.py (74%) rename src/diffusers/pipelines/{ => deprecated}/blip_diffusion/blip_image_processing.py (100%) rename src/diffusers/pipelines/{ => deprecated}/blip_diffusion/modeling_blip2.py (100%) rename src/diffusers/pipelines/{ => deprecated}/blip_diffusion/modeling_ctx_clip.py (100%) rename src/diffusers/pipelines/{ => deprecated}/blip_diffusion/pipeline_blip_diffusion.py (100%) create mode 100644 src/diffusers/pipelines/deprecated/controlnet/__init__.py rename src/diffusers/pipelines/{ => deprecated}/controlnet/pipeline_controlnet_blip_diffusion.py (100%) diff --git a/src/diffusers/pipelines/__init__.py b/src/diffusers/pipelines/__init__.py index 6b714d31c0e3..6cbc2b16e91e 100644 --- a/src/diffusers/pipelines/__init__.py +++ b/src/diffusers/pipelines/__init__.py @@ -146,7 +146,6 @@ "AudioLDM2ProjectionModel", "AudioLDM2UNet2DConditionModel", ] - _import_structure["blip_diffusion"] = ["BlipDiffusionPipeline"] _import_structure["cogvideo"] = [ "CogVideoXPipeline", "CogVideoXImageToVideoPipeline", @@ -158,7 +157,6 @@ _import_structure["consisid"] = ["ConsisIDPipeline"] _import_structure["controlnet"].extend( [ - "BlipDiffusionControlNetPipeline", "StableDiffusionControlNetImg2ImgPipeline", "StableDiffusionControlNetInpaintPipeline", "StableDiffusionControlNetPipeline", @@ -503,7 +501,6 @@ AudioLDM2UNet2DConditionModel, ) from .aura_flow import AuraFlowPipeline - from .blip_diffusion import BlipDiffusionPipeline from .cogvideo import ( CogVideoXFunControlPipeline, CogVideoXImageToVideoPipeline, @@ -514,7 +511,6 @@ from .cogview4 import CogView4ControlPipeline, CogView4Pipeline from .consisid import ConsisIDPipeline from .controlnet import ( - BlipDiffusionControlNetPipeline, StableDiffusionControlNetImg2ImgPipeline, StableDiffusionControlNetInpaintPipeline, StableDiffusionControlNetPipeline, @@ -544,6 +540,8 @@ from .deprecated import ( AltDiffusionImg2ImgPipeline, AltDiffusionPipeline, + BlipDiffusionControlNetPipeline, + BlipDiffusionPipeline, CycleDiffusionPipeline, StableDiffusionInpaintPipelineLegacy, StableDiffusionModelEditingPipeline, diff --git a/src/diffusers/pipelines/controlnet/__init__.py b/src/diffusers/pipelines/controlnet/__init__.py index a49dccf235a3..3400d85d694d 100644 --- a/src/diffusers/pipelines/controlnet/__init__.py +++ b/src/diffusers/pipelines/controlnet/__init__.py @@ -24,7 +24,6 @@ else: _import_structure["multicontrolnet"] = ["MultiControlNetModel"] _import_structure["pipeline_controlnet"] = ["StableDiffusionControlNetPipeline"] - _import_structure["pipeline_controlnet_blip_diffusion"] = ["BlipDiffusionControlNetPipeline"] _import_structure["pipeline_controlnet_img2img"] = ["StableDiffusionControlNetImg2ImgPipeline"] _import_structure["pipeline_controlnet_inpaint"] = ["StableDiffusionControlNetInpaintPipeline"] _import_structure["pipeline_controlnet_inpaint_sd_xl"] = ["StableDiffusionXLControlNetInpaintPipeline"] @@ -54,7 +53,6 @@ else: from .multicontrolnet import MultiControlNetModel from .pipeline_controlnet import StableDiffusionControlNetPipeline - from .pipeline_controlnet_blip_diffusion import BlipDiffusionControlNetPipeline from .pipeline_controlnet_img2img import StableDiffusionControlNetImg2ImgPipeline from .pipeline_controlnet_inpaint import StableDiffusionControlNetInpaintPipeline from .pipeline_controlnet_inpaint_sd_xl import StableDiffusionXLControlNetInpaintPipeline diff --git a/src/diffusers/pipelines/deprecated/__init__.py b/src/diffusers/pipelines/deprecated/__init__.py index 9936323170ad..d8b7e81c4795 100644 --- a/src/diffusers/pipelines/deprecated/__init__.py +++ b/src/diffusers/pipelines/deprecated/__init__.py @@ -42,6 +42,7 @@ "AltDiffusionPipeline", "AltDiffusionPipelineOutput", ] + _import_structure["blip_diffusion"] = ["BlipDiffusionPipeline"] _import_structure["versatile_diffusion"] = [ "VersatileDiffusionDualGuidedPipeline", "VersatileDiffusionImageVariationPipeline", @@ -56,6 +57,7 @@ "StableDiffusionParadigmsPipeline", "StableDiffusionModelEditingPipeline", ] + _import_structure["controlnet"] = ["BlipDiffusionControlNetPipeline"] try: if not (is_torch_available() and is_librosa_available()): @@ -103,6 +105,8 @@ else: from .alt_diffusion import AltDiffusionImg2ImgPipeline, AltDiffusionPipeline, AltDiffusionPipelineOutput from .audio_diffusion import AudioDiffusionPipeline, Mel + from .blip_diffusion import BlipDiffusionPipeline + from .controlnet import BlipDiffusionControlNetPipeline from .spectrogram_diffusion import SpectrogramDiffusionPipeline from .stable_diffusion_variants import ( CycleDiffusionPipeline, diff --git a/src/diffusers/pipelines/blip_diffusion/__init__.py b/src/diffusers/pipelines/deprecated/blip_diffusion/__init__.py similarity index 74% rename from src/diffusers/pipelines/blip_diffusion/__init__.py rename to src/diffusers/pipelines/deprecated/blip_diffusion/__init__.py index af6c879d5ce8..1b8ff411d82e 100644 --- a/src/diffusers/pipelines/blip_diffusion/__init__.py +++ b/src/diffusers/pipelines/deprecated/blip_diffusion/__init__.py @@ -5,14 +5,14 @@ import PIL from PIL import Image -from ...utils import OptionalDependencyNotAvailable, is_torch_available, is_transformers_available +from ....utils import OptionalDependencyNotAvailable, is_torch_available, is_transformers_available try: if not (is_transformers_available() and is_torch_available()): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: - from ...utils.dummy_torch_and_transformers_objects import ShapEPipeline + from ....utils.dummy_torch_and_transformers_objects import ShapEPipeline else: from .blip_image_processing import BlipImageProcessor from .modeling_blip2 import Blip2QFormerModel diff --git a/src/diffusers/pipelines/blip_diffusion/blip_image_processing.py b/src/diffusers/pipelines/deprecated/blip_diffusion/blip_image_processing.py similarity index 100% rename from src/diffusers/pipelines/blip_diffusion/blip_image_processing.py rename to src/diffusers/pipelines/deprecated/blip_diffusion/blip_image_processing.py diff --git a/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py b/src/diffusers/pipelines/deprecated/blip_diffusion/modeling_blip2.py similarity index 100% rename from src/diffusers/pipelines/blip_diffusion/modeling_blip2.py rename to src/diffusers/pipelines/deprecated/blip_diffusion/modeling_blip2.py diff --git a/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py b/src/diffusers/pipelines/deprecated/blip_diffusion/modeling_ctx_clip.py similarity index 100% rename from src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py rename to src/diffusers/pipelines/deprecated/blip_diffusion/modeling_ctx_clip.py diff --git a/src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py b/src/diffusers/pipelines/deprecated/blip_diffusion/pipeline_blip_diffusion.py similarity index 100% rename from src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py rename to src/diffusers/pipelines/deprecated/blip_diffusion/pipeline_blip_diffusion.py diff --git a/src/diffusers/pipelines/deprecated/controlnet/__init__.py b/src/diffusers/pipelines/deprecated/controlnet/__init__.py new file mode 100644 index 000000000000..b869695b8cd8 --- /dev/null +++ b/src/diffusers/pipelines/deprecated/controlnet/__init__.py @@ -0,0 +1,58 @@ +from typing import TYPE_CHECKING + +from ....utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_flax_available, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ....utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_controlnet_blip_diffusion"] = ["BlipDiffusionControlNetPipeline"] +try: + if not (is_transformers_available() and is_flax_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_flax_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_flax_and_transformers_objects)) +else: + _import_structure["pipeline_flax_controlnet"] = ["FlaxStableDiffusionControlNetPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ....utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_controlnet_blip_diffusion import BlipDiffusionControlNetPipeline + + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py b/src/diffusers/pipelines/deprecated/controlnet/pipeline_controlnet_blip_diffusion.py similarity index 100% rename from src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py rename to src/diffusers/pipelines/deprecated/controlnet/pipeline_controlnet_blip_diffusion.py From 028c1d019b26098d65777f25a98fafe7a3bb8147 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 18 Mar 2025 09:56:26 +0530 Subject: [PATCH 2/6] fixes in imports. --- .../blip_diffusion/pipeline_blip_diffusion.py | 10 +++++----- .../controlnet/pipeline_controlnet_blip_diffusion.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/diffusers/pipelines/deprecated/blip_diffusion/pipeline_blip_diffusion.py b/src/diffusers/pipelines/deprecated/blip_diffusion/pipeline_blip_diffusion.py index cbd8bef67945..6f467d339d01 100644 --- a/src/diffusers/pipelines/deprecated/blip_diffusion/pipeline_blip_diffusion.py +++ b/src/diffusers/pipelines/deprecated/blip_diffusion/pipeline_blip_diffusion.py @@ -17,15 +17,15 @@ import torch from transformers import CLIPTokenizer -from ...models import AutoencoderKL, UNet2DConditionModel -from ...schedulers import PNDMScheduler -from ...utils import ( +from ....models import AutoencoderKL, UNet2DConditionModel +from ....schedulers import PNDMScheduler +from ....utils import ( is_torch_xla_available, logging, replace_example_docstring, ) -from ...utils.torch_utils import randn_tensor -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput from .blip_image_processing import BlipImageProcessor from .modeling_blip2 import Blip2QFormerModel from .modeling_ctx_clip import ContextCLIPTextModel diff --git a/src/diffusers/pipelines/deprecated/controlnet/pipeline_controlnet_blip_diffusion.py b/src/diffusers/pipelines/deprecated/controlnet/pipeline_controlnet_blip_diffusion.py index 88c387d48dd2..b3462a740263 100644 --- a/src/diffusers/pipelines/deprecated/controlnet/pipeline_controlnet_blip_diffusion.py +++ b/src/diffusers/pipelines/deprecated/controlnet/pipeline_controlnet_blip_diffusion.py @@ -18,18 +18,18 @@ import torch from transformers import CLIPTokenizer -from ...models import AutoencoderKL, ControlNetModel, UNet2DConditionModel -from ...schedulers import PNDMScheduler -from ...utils import ( +from ....models import AutoencoderKL, ControlNetModel, UNet2DConditionModel +from ....schedulers import PNDMScheduler +from ....utils import ( is_torch_xla_available, logging, replace_example_docstring, ) -from ...utils.torch_utils import randn_tensor +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ..blip_diffusion.blip_image_processing import BlipImageProcessor from ..blip_diffusion.modeling_blip2 import Blip2QFormerModel from ..blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput if is_torch_xla_available(): From 2526d118e39c3fb7f2d7d04eda98fcc083595980 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 18 Mar 2025 10:02:51 +0530 Subject: [PATCH 3/6] import fixes in tests --- tests/pipelines/blipdiffusion/test_blipdiffusion.py | 6 +++--- .../pipelines/controlnet/test_controlnet_blip_diffusion.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pipelines/blipdiffusion/test_blipdiffusion.py b/tests/pipelines/blipdiffusion/test_blipdiffusion.py index e073f55aec9e..684a2dc92a80 100644 --- a/tests/pipelines/blipdiffusion/test_blipdiffusion.py +++ b/tests/pipelines/blipdiffusion/test_blipdiffusion.py @@ -22,10 +22,10 @@ from transformers.models.clip.configuration_clip import CLIPTextConfig from diffusers import AutoencoderKL, BlipDiffusionPipeline, PNDMScheduler, UNet2DConditionModel +from diffusers.pipelines.deprecated.blip_diffusion.blip_image_processing import BlipImageProcessor +from diffusers.pipelines.deprecated.blip_diffusion.modeling_blip2 import Blip2QFormerModel +from diffusers.pipelines.deprecated.blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel from diffusers.utils.testing_utils import enable_full_determinism -from src.diffusers.pipelines.blip_diffusion.blip_image_processing import BlipImageProcessor -from src.diffusers.pipelines.blip_diffusion.modeling_blip2 import Blip2QFormerModel -from src.diffusers.pipelines.blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel from ..test_pipelines_common import PipelineTesterMixin diff --git a/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py b/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py index eedda4e21722..a94be611c870 100644 --- a/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py +++ b/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py @@ -28,10 +28,10 @@ PNDMScheduler, UNet2DConditionModel, ) +from diffusers.pipelines.deprecated.blip_diffusion.blip_image_processing import BlipImageProcessor +from diffusers.pipelines.deprecated.blip_diffusion.modeling_blip2 import Blip2QFormerModel +from diffusers.pipelines.deprecated.blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel from diffusers.utils.testing_utils import enable_full_determinism, torch_device -from src.diffusers.pipelines.blip_diffusion.blip_image_processing import BlipImageProcessor -from src.diffusers.pipelines.blip_diffusion.modeling_blip2 import Blip2QFormerModel -from src.diffusers.pipelines.blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel from ..test_pipelines_common import PipelineTesterMixin From b4fa6f300e5a2b9dc7d5598506aab630a36a65ab Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 18 Mar 2025 10:11:40 +0530 Subject: [PATCH 4/6] fixes --- src/diffusers/pipelines/__init__.py | 1 + tests/pipelines/blipdiffusion/test_blipdiffusion.py | 3 ++- tests/pipelines/controlnet/test_controlnet_blip_diffusion.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/diffusers/pipelines/__init__.py b/src/diffusers/pipelines/__init__.py index 6cbc2b16e91e..9f9113efddb3 100644 --- a/src/diffusers/pipelines/__init__.py +++ b/src/diffusers/pipelines/__init__.py @@ -105,6 +105,7 @@ "VQDiffusionPipeline", "AltDiffusionPipeline", "AltDiffusionImg2ImgPipeline", + "BlipDiffusionPipeline" "BlipDiffusionControlNetPipeline", "CycleDiffusionPipeline", "StableDiffusionInpaintPipelineLegacy", "StableDiffusionPix2PixZeroPipeline", diff --git a/tests/pipelines/blipdiffusion/test_blipdiffusion.py b/tests/pipelines/blipdiffusion/test_blipdiffusion.py index 684a2dc92a80..10baa48b61b0 100644 --- a/tests/pipelines/blipdiffusion/test_blipdiffusion.py +++ b/tests/pipelines/blipdiffusion/test_blipdiffusion.py @@ -21,7 +21,8 @@ from transformers.models.blip_2.configuration_blip_2 import Blip2Config from transformers.models.clip.configuration_clip import CLIPTextConfig -from diffusers import AutoencoderKL, BlipDiffusionPipeline, PNDMScheduler, UNet2DConditionModel +from diffusers import AutoencoderKL, PNDMScheduler, UNet2DConditionModel +from diffusers.pipelines.deprecated import BlipDiffusionPipeline from diffusers.pipelines.deprecated.blip_diffusion.blip_image_processing import BlipImageProcessor from diffusers.pipelines.deprecated.blip_diffusion.modeling_blip2 import Blip2QFormerModel from diffusers.pipelines.deprecated.blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel diff --git a/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py b/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py index a94be611c870..91b239068ec8 100644 --- a/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py +++ b/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py @@ -23,11 +23,11 @@ from diffusers import ( AutoencoderKL, - BlipDiffusionControlNetPipeline, ControlNetModel, PNDMScheduler, UNet2DConditionModel, ) +from diffusers.pipelines.deprecated import BlipDiffusionControlNetPipeline from diffusers.pipelines.deprecated.blip_diffusion.blip_image_processing import BlipImageProcessor from diffusers.pipelines.deprecated.blip_diffusion.modeling_blip2 import Blip2QFormerModel from diffusers.pipelines.deprecated.blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel From 110f89905bbf732540b682e90d211f01a2c112e8 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 18 Mar 2025 10:23:55 +0530 Subject: [PATCH 5/6] more import fixes. --- .../pipelines/deprecated/controlnet/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/diffusers/pipelines/deprecated/controlnet/__init__.py b/src/diffusers/pipelines/deprecated/controlnet/__init__.py index b869695b8cd8..a8024253131e 100644 --- a/src/diffusers/pipelines/deprecated/controlnet/__init__.py +++ b/src/diffusers/pipelines/deprecated/controlnet/__init__.py @@ -5,7 +5,6 @@ OptionalDependencyNotAvailable, _LazyModule, get_objects_from_module, - is_flax_available, is_torch_available, is_transformers_available, ) @@ -23,15 +22,6 @@ _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) else: _import_structure["pipeline_controlnet_blip_diffusion"] = ["BlipDiffusionControlNetPipeline"] -try: - if not (is_transformers_available() and is_flax_available()): - raise OptionalDependencyNotAvailable() -except OptionalDependencyNotAvailable: - from ...utils import dummy_flax_and_transformers_objects # noqa F403 - - _dummy_objects.update(get_objects_from_module(dummy_flax_and_transformers_objects)) -else: - _import_structure["pipeline_flax_controlnet"] = ["FlaxStableDiffusionControlNetPipeline"] if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: From f3fc5f75e422178f7673ad708553cd6902d449ff Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Tue, 18 Mar 2025 13:14:26 +0530 Subject: [PATCH 6/6] Update src/diffusers/pipelines/__init__.py Co-authored-by: hlky --- src/diffusers/pipelines/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/__init__.py b/src/diffusers/pipelines/__init__.py index 9f9113efddb3..488a57c35dc8 100644 --- a/src/diffusers/pipelines/__init__.py +++ b/src/diffusers/pipelines/__init__.py @@ -105,7 +105,8 @@ "VQDiffusionPipeline", "AltDiffusionPipeline", "AltDiffusionImg2ImgPipeline", - "BlipDiffusionPipeline" "BlipDiffusionControlNetPipeline", + "BlipDiffusionPipeline", + "BlipDiffusionControlNetPipeline", "CycleDiffusionPipeline", "StableDiffusionInpaintPipelineLegacy", "StableDiffusionPix2PixZeroPipeline",