can't load lora weights after apply group offloading

[zhangvia]

Describe the bug

i want to dynamically switch the lora with applying group offloading ,but failed

Reproduction

from diffusers import HunyuanVideoPipeline, HunyuanVideoTransformer3DModel
from diffusers.utils import export_to_video
import torch
from diffusers.hooks import apply_group_offloading


model_id = "HunyuanVideo-Diffusers/"
transformer = HunyuanVideoTransformer3DModel.from_pretrained(
    model_id, subfolder="transformer", torch_dtype=torch.bfloat16
)
pipe = HunyuanVideoPipeline.from_pretrained(model_id, transformer=transformer, torch_dtype=torch.float16)
pipe.vae.enable_tiling()
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
pipe.load_lora_weights("a-r-r-o-w/HunyuanVideo-tuxemons/")

list(map(
    lambda x: apply_group_offloading(x, onload_device, offload_device, offload_type="block_level",num_blocks_per_group=1,use_stream=True,non_blocking=True),
    [pipe.text_encoder,pipe.text_encoder_2,pipe.transformer]
))
pipe.vae.to('cuda')


output = pipe(
    prompt="Style of snomexut, a cat-like Tuxemon creature walks in alien-world grass, and observes its surroundings.",
    height=768,
    width=768,
    num_frames=33,
    num_inference_steps=30,
    generator=torch.Generator().manual_seed(73),
).frames[0]
export_to_video(output, "output-tuxemon.mp4", fps=15)



pipe.unload_lora_weights()

output_1 = pipe(
    prompt="Style of snomexut, a cat-like Tuxemon creature walks in alien-world grass, and observes its surroundings.",
    height=768,
    width=768,
    num_frames=33,
    num_inference_steps=30,
    generator=torch.Generator().manual_seed(73),
).frames[0]
export_to_video(output_1, "output-tuxemon-1.mp4", fps=15)


pipe.load_lora_weights("a-r-r-o-w/HunyuanVideo-tuxemons/")
output_2 = pipe(
    prompt="Style of snomexut, a cat-like Tuxemon creature walks in alien-world grass, and observes its surroundings.",
    height=768,
    width=768,
    num_frames=33,
    num_inference_steps=30,
    generator=torch.Generator().manual_seed(73),
).frames[0]
export_to_video(output_2, "output-tuxemon-2.mp4", fps=15)

log:

Logs

System Info

diffusers 0.34.0.dev0

Who can help?

@a-r-r-o-w