add flux inpaint + img2img + controlnet to auto pipeline

[yiyixuxu]

this PR

1. add the flux inpaint + img2img + controlnet to autopipeline
2. refactored AutoPipelineforInpaint.from_pretrained a little bit so it is clearer, and will work with controlnet when we have a controlnet +pag+inpaint pipeline available (Currently, we don't have one yet!). this is a follow-up for Fix StableDiffusionXLPAGInpaintPipeline #9128
3. similarly, made AutoPipelineForImage2Image work with the refiner checkpoint
4. add a little bit more test (mostly add a test to test AutoPipelineforInpaint.from_pretrained from inpaint-specific checkpoint, the only meaningful use case for that is to use with PAG) and also AutoPipelineforInpaint.from_pretrained from refiner checkpoint

testing script for everything I've added in this PR

from diffusers import AutoPipelineForInpainting, AutoPipelineForImage2Image, AutoPipelineForText2Image, FluxControlNetModel, ControlNetModel
import torch

torch.cuda.set_device(1)

# flux inpaint
pipe = AutoPipelineForInpainting.from_pretrained("black-forest-labs/FLUX.1-schnell")
assert pipe.__class__.__name__ == "FluxInpaintPipeline"

# flux img2img
pipe = AutoPipelineForImage2Image.from_pretrained("black-forest-labs/FLUX.1-schnell")
assert pipe.__class__.__name__ == "FluxImg2ImgPipeline"

# Flux ControlNet
controlnet_model = "InstantX/FLUX.1-dev-controlnet-canny"
controlnet = FluxControlNetModel.from_pretrained(controlnet_model, torch_dtype=torch.bfloat16)

pipe = AutoPipelineForText2Image.from_pretrained("black-forest-labs/FLUX.1-dev", controlnet=controlnet, torch_dtype=torch.bfloat16)
assert pipe.__class__.__name__ == "FluxControlNetPipeline"

# AutoPipelineForInpainting
inpaint_checkpoint = "diffusers/stable-diffusion-xl-1.0-inpainting-0.1"
base_checkpoint = "stabilityai/stable-diffusion-xl-base-1.0"


pipe = AutoPipelineForInpainting.from_pretrained(inpaint_checkpoint, enable_pag=True,torch_dtype=torch.float16)
assert pipe.__class__.__name__ == "StableDiffusionXLPAGInpaintPipeline"

pipe = AutoPipelineForInpainting.from_pretrained(base_checkpoint, enable_pag=True,torch_dtype=torch.float16)
assert pipe.__class__.__name__ == "StableDiffusionXLPAGInpaintPipeline"

pipe = AutoPipelineForInpainting.from_pretrained(inpaint_checkpoint, torch_dtype=torch.float16)
assert pipe.__class__.__name__ == "StableDiffusionXLInpaintPipeline"

pipe = AutoPipelineForInpainting.from_pretrained(base_checkpoint, torch_dtype=torch.float16)
assert pipe.__class__.__name__ == "StableDiffusionXLInpaintPipeline"


# test AutoPipelineForImage2Image.from_pretrained, make sure it can load both text2img checkpoint and img2img checkpoint (refiner)
controlnet = ControlNetModel.from_pretrained(
    "diffusers/controlnet-depth-sdxl-1.0-small",
    variant="fp16",
    torch_dtype=torch.float16,
)

# -> img2img + pag
refiner_checkpoint = "stabilityai/stable-diffusion-xl-refiner-1.0"
base_checkpoint = "stabilityai/stable-diffusion-xl-base-1.0"

## refiner checkpoint (img2img) -> img2img + pag
pipe = AutoPipelineForImage2Image.from_pretrained(
    refiner_checkpoint,
    torch_dtype=torch.float16,
    enable_pag=True,
)
assert pipe.__class__.__name__ == "StableDiffusionXLPAGImg2ImgPipeline"

## base checkpoint (text2img) -> img2img + pag
pipe = AutoPipelineForImage2Image.from_pretrained(
    base_checkpoint,
    torch_dtype=torch.float16,
    enable_pag=True,
)
assert pipe.__class__.__name__ == "StableDiffusionXLPAGImg2ImgPipeline"

# -> img2img + controlne

## refiner checkpoint (img2img) -> img2img + controlnet
pipe = AutoPipelineForImage2Image.from_pretrained(
    refiner_checkpoint,
    controlnet=controlnet,
    torch_dtype=torch.float16,
)
assert pipe.__class__.__name__ == "StableDiffusionXLControlNetImg2ImgPipeline"

## base checkpoint (text2img) -> img2img + controlnet
pipe = AutoPipelineForImage2Image.from_pretrained(
    base_checkpoint,
    controlnet=controlnet,
    torch_dtype=torch.float16,
)
assert pipe.__class__.__name__ == "StableDiffusionXLControlNetImg2ImgPipeline"


# -> img2img + controlnet + pag

## refiner checkpoint (img2img) -> img2img + controlnet + pag
pipe = AutoPipelineForImage2Image.from_pretrained(
    refiner_checkpoint,
    controlnet=controlnet,
    torch_dtype=torch.float16,
    enable_pag=True,
)
assert pipe.__class__.__name__ == "StableDiffusionXLControlNetPAGImg2ImgPipeline"

## base checkpoint (text2img) -> img2img + controlnet + pag
pipe = AutoPipelineForImage2Image.from_pretrained(
    base_checkpoint,
    controlnet=controlnet,
    torch_dtype=torch.float16,
    enable_pag=True,
)
assert pipe.__class__.__name__ == "StableDiffusionXLControlNetPAGImg2ImgPipeline"

[HuggingFaceDocBuilderDev]

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.