LTX 0.9.5

[a-r-r-o-w]

See https://huggingface.slack.com/archives/C08275HSG8J/p1741091747532049?thread_ts=1738246363.413529&cid=C08275HSG8J

testing scripts

test1: image condition

# test ltx image conditioning
import torch
from diffusers import LTXConditionPipeline
from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition
from diffusers.utils import export_to_video, load_video, load_image


device = "cuda:2"
dtype = torch.bfloat16
repo = "YiYiXu/ltx-95"

# Initialize the pipeline
pipe = LTXConditionPipeline.from_pretrained(repo, torch_dtype=dtype)
pipe.to(device)


prompt = "The video depicts a long, straight highway stretching into the distance, flanked by metal guardrails. The road is divided into multiple lanes, with a few vehicles visible in the far distance. The surrounding landscape features dry, grassy fields on one side and rolling hills on the other. The sky is mostly clear with a few scattered clouds, suggesting a bright, sunny day."
negative_prompt='worst quality, inconsistent motion, blurry, jittery, distorted'
image = load_image(
    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cosmos/cosmos-video2world-input.jpg"
)

# use `conditions` input
condition = LTXVideoCondition(
    image=image,
)

generator = torch.Generator(device=device).manual_seed(0)
video = pipe(
    conditions=[condition],
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    num_inference_steps=40,
    generator=generator,
).frames[0]

# Export the video
export_to_video(video, "yiyi_test_1_output_1.mp4", fps=24)


#  pass `image` input directly
generator = torch.Generator(device=device).manual_seed(0)
video = pipe(
    image=image,
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    num_inference_steps=40,
    generator=generator,
).frames[0]

# Export the video
export_to_video(video, "yiyi_test_1_output_2.mp4", fps=24)

test2: video condition

# ltx video conditioning
import torch
from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXConditionPipeline, LTXVideoCondition
from diffusers.utils import export_to_video, load_video


device = "cuda:2"
dtype = torch.bfloat16
repo = "YiYiXu/ltx-95"

# Initialize the pipeline
pipe = LTXConditionPipeline.from_pretrained(repo, torch_dtype=dtype)
pipe.to(device)


video = load_video(
    "/raid/yiyi/LTX-Video/outputs/2025-03-11/video_output_0_a-woman-with-long-brown-hair-and_42_512x768x40_0.mp4"
)

condition = LTXVideoCondition(
    video=video,
    frame_index=0
)

# Define prompts
prompt = "A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage"
negative_prompt='worst quality, inconsistent motion, blurry, jittery, distorted'
# Generate the video
video = pipe(
    conditions=[condition],
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    num_inference_steps=40,
).frames[0]

# Export the video
export_to_video(video, "yiyi_test_2_output_1.mp4", fps=24)



# test 2: frame index 8
condition = LTXVideoCondition(
    video=video,
    frame_index=8
)

# Generate the video
video = pipe(
    conditions=[condition],
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    num_inference_steps=40,
).frames[0]

# Export the video
export_to_video(video, "yiyi_test_2_output_2.mp4", fps=24)


# test3: pass inputs directly

# Generate the video
video = pipe(
    video=video,
    frame_index=8,
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    num_inference_steps=40,
).frames[0]

# Export the video
export_to_video(video, "yiyi_test_2_output_3.mp4", fps=24)

test3: video + image

# image + video
import torch
from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXConditionPipeline, LTXVideoCondition
from diffusers.utils import export_to_video, load_video, load_image


device = "cuda:2"
dtype = torch.bfloat16
repo = "YiYiXu/ltx-95"

# Initialize the pipeline
pipe = LTXConditionPipeline.from_pretrained(repo, torch_dtype=dtype)
pipe.to(device)


video = load_video(
    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cosmos/cosmos-video2world-input-vid.mp4"
)

image = load_image(
    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cosmos/cosmos-video2world-input.jpg"
)

condition1 = LTXVideoCondition(
    image=image,
    frame_index=0,
)

condition2 = LTXVideoCondition(
    video=video,
    frame_index=80,
)

# Define prompts
prompt = "The video depicts a long, straight highway stretching into the distance, flanked by metal guardrails. The road is divided into multiple lanes, with a few vehicles visible in the far distance. The surrounding landscape features dry, grassy fields on one side and rolling hills on the other. The sky is mostly clear with a few scattered clouds, suggesting a bright, sunny day. And then the camera switch to a inding mountain road covered in snow, with a single vehicle traveling along it. The road is flanked by steep, rocky cliffs and sparse vegetation. The landscape is characterized by rugged terrain and a river visible in the distance. The scene captures the solitude and beauty of a winter drive through a mountainous region."
negative_prompt='worst quality, inconsistent motion, blurry, jittery, distorted'
# Generate the video
generator = torch.Generator(device=device).manual_seed(0)
video = pipe(
    conditions=[condition1, condition2],
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    num_inference_steps=40,
    generator=generator,
).frames[0]

# Export the video
export_to_video(video, "yiyi_test_4_output.mp4", fps=24)

[HuggingFaceDocBuilderDev]

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

[a-r-r-o-w]

Code for matching VAE:

import sys
sys.path.append("/raid/aryan/ltx-code")

import json
from typing import Any, Dict

import torch
from safetensors.torch import load_file
from safetensors import safe_open

from ltx_video.models.autoencoders.causal_video_autoencoder import CausalVideoAutoencoder


def remove_keys_(key: str, state_dict: Dict[str, Any]):
    state_dict.pop(key)

VAE_KEYS_RENAME_DICT = {
    # decoder
    "up_blocks.0": "mid_block",
    "up_blocks.1": "up_blocks.0",
    "up_blocks.2": "up_blocks.1.upsamplers.0",
    "up_blocks.3": "up_blocks.1",
    "up_blocks.4": "up_blocks.2.conv_in",
    "up_blocks.5": "up_blocks.2.upsamplers.0",
    "up_blocks.6": "up_blocks.2",
    "up_blocks.7": "up_blocks.3.conv_in",
    "up_blocks.8": "up_blocks.3.upsamplers.0",
    "up_blocks.9": "up_blocks.3",
    # encoder
    "down_blocks.0": "down_blocks.0",
    "down_blocks.1": "down_blocks.0.downsamplers.0",
    "down_blocks.2": "down_blocks.0.conv_out",
    "down_blocks.3": "down_blocks.1",
    "down_blocks.4": "down_blocks.1.downsamplers.0",
    "down_blocks.5": "down_blocks.1.conv_out",
    "down_blocks.6": "down_blocks.2",
    "down_blocks.7": "down_blocks.2.downsamplers.0",
    "down_blocks.8": "down_blocks.3",
    "down_blocks.9": "mid_block",
    # common
    "conv_shortcut": "conv_shortcut.conv",
    "res_blocks": "resnets",
    "norm3.norm": "norm3",
    "per_channel_statistics.mean-of-means": "latents_mean",
    "per_channel_statistics.std-of-means": "latents_std",
}

VAE_091_RENAME_DICT = {
    # decoder
    "up_blocks.0": "mid_block",
    "up_blocks.1": "up_blocks.0.upsamplers.0",
    "up_blocks.2": "up_blocks.0",
    "up_blocks.3": "up_blocks.1.upsamplers.0",
    "up_blocks.4": "up_blocks.1",
    "up_blocks.5": "up_blocks.2.upsamplers.0",
    "up_blocks.6": "up_blocks.2",
    "up_blocks.7": "up_blocks.3.upsamplers.0",
    "up_blocks.8": "up_blocks.3",
    # common
    "last_time_embedder": "time_embedder",
    "last_scale_shift_table": "scale_shift_table",
}

VAE_095_RENAME_DICT = {
    # decoder
    "up_blocks.0": "mid_block",
    "up_blocks.1": "up_blocks.0.upsamplers.0",
    "up_blocks.2": "up_blocks.0",
    "up_blocks.3": "up_blocks.1.upsamplers.0",
    "up_blocks.4": "up_blocks.1",
    "up_blocks.5": "up_blocks.2.upsamplers.0",
    "up_blocks.6": "up_blocks.2",
    "up_blocks.7": "up_blocks.3.upsamplers.0",
    "up_blocks.8": "up_blocks.3",
    # encoder
    "down_blocks.0": "down_blocks.0",
    "down_blocks.1": "down_blocks.0.downsamplers.0",
    "down_blocks.2": "down_blocks.1",
    "down_blocks.3": "down_blocks.1.downsamplers.0",
    "down_blocks.4": "down_blocks.2",
    "down_blocks.5": "down_blocks.2.downsamplers.0",
    "down_blocks.6": "down_blocks.3",
    "down_blocks.7": "down_blocks.3.downsamplers.0",
    "down_blocks.8": "mid_block",
    # common
    "last_time_embedder": "time_embedder",
    "last_scale_shift_table": "scale_shift_table",
}

VAE_SPECIAL_KEYS_REMAP = {
    "per_channel_statistics.channel": remove_keys_,
    "per_channel_statistics.mean-of-means": remove_keys_,
    "per_channel_statistics.mean-of-stds": remove_keys_,
    "model.diffusion_model": remove_keys_,
}

VAE_091_SPECIAL_KEYS_REMAP = {
    "timestep_scale_multiplier": remove_keys_,
}

VAE_095_SPECIAL_KEYS_REMAP = {
    
}


def update_state_dict_inplace(state_dict: Dict[str, Any], old_key: str, new_key: str) -> Dict[str, Any]:
    state_dict[new_key] = state_dict.pop(old_key)


def convert_vae(original_state_dict):
    PREFIX_KEY = "vae."

    for key in list(original_state_dict.keys()):
        new_key = key[:]
        if new_key.startswith(PREFIX_KEY):
            new_key = key[len(PREFIX_KEY) :]
        for replace_key, rename_key in VAE_KEYS_RENAME_DICT.items():
            new_key = new_key.replace(replace_key, rename_key)
        update_state_dict_inplace(original_state_dict, key, new_key)

    for key in list(original_state_dict.keys()):
        for special_key, handler_fn_inplace in VAE_SPECIAL_KEYS_REMAP.items():
            if special_key not in key:
                continue
            handler_fn_inplace(key, original_state_dict)

    return original_state_dict


@torch.no_grad()
def match_vae():
    from diffusers import AutoencoderKLLTXVideo

    original_model_path = "/raid/aryan/ltx-new/ltx-video-2b-v0.9.5rc1.safetensors"
    theirs_config = json.loads(safe_open(original_model_path, "pt").metadata()["config"])
    theirs_model = CausalVideoAutoencoder.from_config(theirs_config["vae"])
    theirs_state_dict = load_file(original_model_path)
    theirs_model.load_state_dict(theirs_state_dict)

    ours_config = {
        "in_channels": 3,
        "out_channels": 3,
        "latent_channels": 128,
        "block_out_channels": (128, 256, 512, 1024, 2048),
        "down_block_types": (
            "LTXVideo095DownBlock3D",
            "LTXVideo095DownBlock3D",
            "LTXVideo095DownBlock3D",
            "LTXVideo095DownBlock3D",
        ),
        "decoder_block_out_channels": (256, 512, 1024),
        "layers_per_block": (4, 6, 6, 2, 2),
        "decoder_layers_per_block": (5, 5, 5, 5),
        "spatio_temporal_scaling": (True, True, True, True),
        "decoder_spatio_temporal_scaling": (True, True, True),
        "decoder_inject_noise": (False, False, False, False),
        "downsample_type": ("spatial", "temporal", "spatiotemporal", "spatiotemporal"),
        "upsample_residual": (True, True, True),
        "upsample_factor": (2, 2, 2),
        "timestep_conditioning": True,
        "patch_size": 4,
        "patch_size_t": 1,
        "resnet_norm_eps": 1e-6,
        "scaling_factor": 1.0,
        "encoder_causal": True,
        "decoder_causal": False,
    }
    ours_model = AutoencoderKLLTXVideo.from_config(ours_config)

    VAE_KEYS_RENAME_DICT.update(VAE_095_RENAME_DICT)
    VAE_SPECIAL_KEYS_REMAP.update(VAE_095_SPECIAL_KEYS_REMAP)
    ours_state_dict = convert_vae(theirs_state_dict)
    ours_model.load_state_dict(ours_state_dict)

    state_dict_params = sum(p.numel() for p in ours_state_dict.values())
    print(f"State dict params: {state_dict_params}")

    device = torch.device("cuda")
    dtype = torch.float32

    theirs_model.to(device=device, dtype=dtype)
    ours_model.to(device=device, dtype=dtype)

    theirs_model.disable_z_tiling()
    theirs_model.disable_hw_tiling()

    print(sum(p.numel() for p in theirs_model.parameters()))
    print(sum(p.numel() for p in ours_model.parameters()))

    batch_size = 1
    num_channels = 3
    num_frames = 49
    height = 128
    width = 128

    torch.manual_seed(0)
    input = torch.randn(batch_size, num_channels, num_frames, height, width, device=device, dtype=dtype)
    decode_timestep = 0.025

    print("theirs_encoding")
    theirs_encoder_output = theirs_model.encode(input).latent_dist.mode()
    print("theirs_decoding")
    theirs_decoder_output = theirs_model.decode(theirs_encoder_output, timestep=decode_timestep, target_shape=(batch_size, num_channels, num_frames, height, width)).sample
    print("theirs:", theirs_encoder_output.shape, theirs_decoder_output.shape)

    print("ours_encoding")
    ours_encoder_output = ours_model.encode(input).latent_dist.mode()
    print("ours_decoding")
    ours_decoder_output = ours_model.decode(ours_encoder_output, temb=decode_timestep).sample
    print("ours:", ours_encoder_output.shape, ours_decoder_output.shape)

    diff_encoder = theirs_encoder_output - ours_encoder_output
    diff_decoder = theirs_decoder_output - ours_decoder_output

    absmax_encoder, absmean_encoder = torch.max(diff_encoder.abs()), torch.mean(diff_encoder.abs())
    absmax_decoder, absmean_decoder = torch.max(diff_decoder.abs()), torch.mean(diff_decoder.abs())

    print(f"Encoder: absmax={absmax_encoder}, absmean={absmean_encoder}")
    print(f"Decoder: absmax={absmax_decoder}, absmean={absmean_decoder}")


match_vae()

[yiyixuxu]

I like the LTXVideoCondition class!
but let's still support the image/video inputs still so that users can do a simple image2video video2video with the same API they use for other img2video/video2video

[a-r-r-o-w]

Sounds good!

[a-r-r-o-w]

Thanks @yiyixuxu! Just had a question, but otherwise looks good to merge

[a-r-r-o-w]

Sounds good!

[SHYuanBest]

Hi, it seem that LTXPipeline have a bug when loading LTX-0.9.5 (to device)?

import torch
from diffusers import LTXPipeline
from diffusers.utils import export_to_video

base_model_path = "YiYiXu/ltx-95"

pipe = LTXPipeline.from_pretrained(base_model_path, torch_dtype=torch.bfloat16)
pipe.to("cuda")

The config attributes {'timestep_scale_multiplier': 1000.0} were passed to AutoencoderKLLTXVideo, but are not expected and will be ignored. Please verify your config.json configuration file.
Some weights of AutoencoderKLLTXVideo were not initialized from the model checkpoint atLTX-Video-0.9.5-diffusers/vae and are newly initialized: ['decoder.timestep_scale_multiplier']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Loading pipeline components...: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:02<00:00,  2.07it/s]
Traceback (most recent call last):
  File "inference.py", line 8, in <module>
    pipe.to("cuda")
  File "/storage/miniconda3/envs/wan_train/lib/python3.11/site-packages/diffusers/pipelines/pipeline_utils.py", line 482, in to
    module.to(device, dtype)
  File "/storage/miniconda3/envs/wan_train/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 1351, in to
    return super().to(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/storage/miniconda3/envs/wan_train/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1340, in to
    return self._apply(convert)
           ^^^^^^^^^^^^^^^^^^^^
  File "/storage/miniconda3/envs/wan_train/lib/python3.11/site-packages/torch/nn/modules/module.py", line 900, in _apply
    module._apply(fn)
  File "/storage/miniconda3/envs/wan_train/lib/python3.11/site-packages/torch/nn/modules/module.py", line 927, in _apply
    param_applied = fn(param)
                    ^^^^^^^^^
  File "/storage/miniconda3/envs/wan_train/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1333, in convert
    raise NotImplementedError(
NotImplementedError: Cannot copy out of meta tensor; no data! Please use torch.nn.Module.to_empty() instead of torch.nn.Module.to() when moving module from meta to a different device.

[a-r-r-o-w]

Hi, please use the official Lightricks repository: https://huggingface.co/Lightricks/LTX-Video-0.9.5