ltx_video/schedulers/rf.py

import math
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Callable, Optional, Tuple, Union
import json
import os
from pathlib import Path

import torch
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.schedulers.scheduling_utils import SchedulerMixin
from diffusers.utils import BaseOutput
from torch import Tensor
from safetensors import safe_open


from ltx_video.utils.torch_utils import append_dims

from ltx_video.utils.diffusers_config_mapping import (
    diffusers_and_ours_config_mapping,
    make_hashable_key,
)


def linear_quadratic_schedule(num_steps, threshold_noise=0.025, linear_steps=None):
    if num_steps == 1:
        return torch.tensor([1.0])
    if linear_steps is None:
        linear_steps = num_steps // 2
    linear_sigma_schedule = [
        i * threshold_noise / linear_steps for i in range(linear_steps)
    ]
    threshold_noise_step_diff = linear_steps - threshold_noise * num_steps
    quadratic_steps = num_steps - linear_steps
    quadratic_coef = threshold_noise_step_diff / (linear_steps * quadratic_steps**2)
    linear_coef = threshold_noise / linear_steps - 2 * threshold_noise_step_diff / (
        quadratic_steps**2
    )
    const = quadratic_coef * (linear_steps**2)
    quadratic_sigma_schedule = [
        quadratic_coef * (i**2) + linear_coef * i + const
        for i in range(linear_steps, num_steps)
    ]
    sigma_schedule = linear_sigma_schedule + quadratic_sigma_schedule + [1.0]
    sigma_schedule = [1.0 - x for x in sigma_schedule]
    return torch.tensor(sigma_schedule[:-1])


def simple_diffusion_resolution_dependent_timestep_shift(
    samples_shape: torch.Size,
    timesteps: Tensor,
    n: int = 32 * 32,
) -> Tensor:
    if len(samples_shape) == 3:
        _, m, _ = samples_shape
    elif len(samples_shape) in [4, 5]:
        m = math.prod(samples_shape[2:])
    else:
        raise ValueError(
            "Samples must have shape (b, t, c), (b, c, h, w) or (b, c, f, h, w)"
        )
    snr = (timesteps / (1 - timesteps)) ** 2
    shift_snr = torch.log(snr) + 2 * math.log(m / n)
    shifted_timesteps = torch.sigmoid(0.5 * shift_snr)

    return shifted_timesteps


def time_shift(mu: float, sigma: float, t: Tensor):
    return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma)


def get_normal_shift(
    n_tokens: int,
    min_tokens: int = 1024,
    max_tokens: int = 4096,
    min_shift: float = 0.95,
    max_shift: float = 2.05,
) -> Callable[[float], float]:
    m = (max_shift - min_shift) / (max_tokens - min_tokens)
    b = min_shift - m * min_tokens
    return m * n_tokens + b


def strech_shifts_to_terminal(shifts: Tensor, terminal=0.1):
    """
    Stretch a function (given as sampled shifts) so that its final value matches the given terminal value
    using the provided formula.

    Parameters:
    - shifts (Tensor): The samples of the function to be stretched (PyTorch Tensor).
    - terminal (float): The desired terminal value (value at the last sample).

    Returns:
    - Tensor: The stretched shifts such that the final value equals `terminal`.
    """
    if shifts.numel() == 0:
        raise ValueError("The 'shifts' tensor must not be empty.")

    # Ensure terminal value is valid
    if terminal <= 0 or terminal >= 1:
        raise ValueError("The terminal value must be between 0 and 1 (exclusive).")

    # Transform the shifts using the given formula
    one_minus_z = 1 - shifts
    scale_factor = one_minus_z[-1] / (1 - terminal)
    stretched_shifts = 1 - (one_minus_z / scale_factor)

    return stretched_shifts


def sd3_resolution_dependent_timestep_shift(
    samples_shape: torch.Size,
    timesteps: Tensor,
    target_shift_terminal: Optional[float] = None,
) -> Tensor:
    """
    Shifts the timestep schedule as a function of the generated resolution.

    In the SD3 paper, the authors empirically how to shift the timesteps based on the resolution of the target images.
    For more details: https://arxiv.org/pdf/2403.03206

    In Flux they later propose a more dynamic resolution dependent timestep shift, see:
    https://github.com/black-forest-labs/flux/blob/87f6fff727a377ea1c378af692afb41ae84cbe04/src/flux/sampling.py#L66


    Args:
        samples_shape (torch.Size): The samples batch shape (batch_size, channels, height, width) or
            (batch_size, channels, frame, height, width).
        timesteps (Tensor): A batch of timesteps with shape (batch_size,).
        target_shift_terminal (float): The target terminal value for the shifted timesteps.

    Returns:
        Tensor: The shifted timesteps.
    """
    if len(samples_shape) == 3:
        _, m, _ = samples_shape
    elif len(samples_shape) in [4, 5]:
        m = math.prod(samples_shape[2:])
    else:
        raise ValueError(
            "Samples must have shape (b, t, c), (b, c, h, w) or (b, c, f, h, w)"
        )

    shift = get_normal_shift(m)
    time_shifts = time_shift(shift, 1, timesteps)
    if target_shift_terminal is not None:  # Stretch the shifts to the target terminal
        time_shifts = strech_shifts_to_terminal(time_shifts, target_shift_terminal)
    return time_shifts


class TimestepShifter(ABC):
    @abstractmethod
    def shift_timesteps(self, samples_shape: torch.Size, timesteps: Tensor) -> Tensor:
        pass


@dataclass
class RectifiedFlowSchedulerOutput(BaseOutput):
    """
    Output class for the scheduler's step function output.

    Args:
        prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
            Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the
            denoising loop.
        pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
            The predicted denoised sample (x_{0}) based on the model output from the current timestep.
            `pred_original_sample` can be used to preview progress or for guidance.
    """

    prev_sample: torch.FloatTensor
    pred_original_sample: Optional[torch.FloatTensor] = None


class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
    order = 1

    @register_to_config
    def __init__(
        self,
        num_train_timesteps=1000,
        shifting: Optional[str] = None,
        base_resolution: int = 32**2,
        target_shift_terminal: Optional[float] = None,
        sampler: Optional[str] = "Uniform",
        shift: Optional[float] = None,
    ):
        super().__init__()
        self.init_noise_sigma = 1.0
        self.num_inference_steps = None
        self.sampler = sampler
        self.shifting = shifting
        self.base_resolution = base_resolution
        self.target_shift_terminal = target_shift_terminal
        self.timesteps = self.sigmas = self.get_initial_timesteps(
            num_train_timesteps, shift=shift
        )
        self.shift = shift

    def get_initial_timesteps(
        self, num_timesteps: int, shift: Optional[float] = None
    ) -> Tensor:
        if self.sampler == "Uniform":
            return torch.linspace(1, 1 / num_timesteps, num_timesteps)
        elif self.sampler == "LinearQuadratic":
            return linear_quadratic_schedule(num_timesteps)
        elif self.sampler == "Constant":
            assert (
                shift is not None
            ), "Shift must be provided for constant time shift sampler."
            return time_shift(
                shift, 1, torch.linspace(1, 1 / num_timesteps, num_timesteps)
            )

    def shift_timesteps(self, samples_shape: torch.Size, timesteps: Tensor) -> Tensor:
        if self.shifting == "SD3":
            return sd3_resolution_dependent_timestep_shift(
                samples_shape, timesteps, self.target_shift_terminal
            )
        elif self.shifting == "SimpleDiffusion":
            return simple_diffusion_resolution_dependent_timestep_shift(
                samples_shape, timesteps, self.base_resolution
            )
        return timesteps

    def set_timesteps(
        self,
        num_inference_steps: Optional[int] = None,
        samples_shape: Optional[torch.Size] = None,
        timesteps: Optional[Tensor] = None,
        device: Union[str, torch.device] = None,
    ):
        """
        Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference.
        If `timesteps` are provided, they will be used instead of the scheduled timesteps.

        Args:
            num_inference_steps (`int` *optional*): The number of diffusion steps used when generating samples.
            samples_shape (`torch.Size` *optional*): The samples batch shape, used for shifting.
            timesteps ('torch.Tensor' *optional*): Specific timesteps to use instead of scheduled timesteps.
            device (`Union[str, torch.device]`, *optional*): The device to which the timesteps tensor will be moved.
        """
        if timesteps is not None and num_inference_steps is not None:
            raise ValueError(
                "You cannot provide both `timesteps` and `num_inference_steps`."
            )
        if timesteps is None:
            num_inference_steps = min(
                self.config.num_train_timesteps, num_inference_steps
            )
            timesteps = self.get_initial_timesteps(
                num_inference_steps, shift=self.shift
            ).to(device)
            timesteps = self.shift_timesteps(samples_shape, timesteps)
        else:
            timesteps = torch.Tensor(timesteps).to(device)
            num_inference_steps = len(timesteps)
        self.timesteps = timesteps
        self.num_inference_steps = num_inference_steps
        self.sigmas = self.timesteps

    @staticmethod
    def from_pretrained(pretrained_model_path: Union[str, os.PathLike]):
        pretrained_model_path = Path(pretrained_model_path)
        if pretrained_model_path.is_file():
            comfy_single_file_state_dict = {}
            with safe_open(pretrained_model_path, framework="pt", device="cpu") as f:
                metadata = f.metadata()
                for k in f.keys():
                    comfy_single_file_state_dict[k] = f.get_tensor(k)
            configs = json.loads(metadata["config"])
            config = configs["scheduler"]
            del comfy_single_file_state_dict

        elif pretrained_model_path.is_dir():
            diffusers_noise_scheduler_config_path = (
                pretrained_model_path / "scheduler" / "scheduler_config.json"
            )

            with open(diffusers_noise_scheduler_config_path, "r") as f:
                scheduler_config = json.load(f)
            hashable_config = make_hashable_key(scheduler_config)
            if hashable_config in diffusers_and_ours_config_mapping:
                config = diffusers_and_ours_config_mapping[hashable_config]
        return RectifiedFlowScheduler.from_config(config)

    def scale_model_input(
        self, sample: torch.FloatTensor, timestep: Optional[int] = None
    ) -> torch.FloatTensor:
        # pylint: disable=unused-argument
        """
        Ensures interchangeability with schedulers that need to scale the denoising model input depending on the
        current timestep.

        Args:
            sample (`torch.FloatTensor`): input sample
            timestep (`int`, optional): current timestep

        Returns:
            `torch.FloatTensor`: scaled input sample
        """
        return sample

    def step(
        self,
        model_output: torch.FloatTensor,
        timestep: torch.FloatTensor,
        sample: torch.FloatTensor,
        return_dict: bool = True,
        stochastic_sampling: Optional[bool] = False,
        **kwargs,
    ) -> Union[RectifiedFlowSchedulerOutput, Tuple]:
        """
        Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
        process from the learned model outputs (most often the predicted noise).
        z_{t_1} = z_t - \Delta_t * v
        The method finds the next timestep that is lower than the input timestep(s) and denoises the latents
        to that level. The input timestep(s) are not required to be one of the predefined timesteps.

        Args:
            model_output (`torch.FloatTensor`):
                The direct output from learned diffusion model - the velocity,
            timestep (`float`):
                The current discrete timestep in the diffusion chain (global or per-token).
            sample (`torch.FloatTensor`):
                A current latent tokens to be de-noised.
            return_dict (`bool`, *optional*, defaults to `True`):
                Whether or not to return a [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] or `tuple`.
            stochastic_sampling (`bool`, *optional*, defaults to `False`):
                Whether to use stochastic sampling for the sampling process.

        Returns:
            [`~schedulers.scheduling_utils.RectifiedFlowSchedulerOutput`] or `tuple`:
                If return_dict is `True`, [`~schedulers.rf_scheduler.RectifiedFlowSchedulerOutput`] is returned,
                otherwise a tuple is returned where the first element is the sample tensor.
        """
        if self.num_inference_steps is None:
            raise ValueError(
                "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"
            )
        t_eps = 1e-6  # Small epsilon to avoid numerical issues in timestep values

        timesteps_padded = torch.cat(
            [self.timesteps, torch.zeros(1, device=self.timesteps.device)]
        )

        # Find the next lower timestep(s) and compute the dt from the current timestep(s)
        if timestep.ndim == 0:
            # Global timestep case
            lower_mask = timesteps_padded < timestep - t_eps
            lower_timestep = timesteps_padded[lower_mask][0]  # Closest lower timestep
            dt = timestep - lower_timestep

        else:
            # Per-token case
            assert timestep.ndim == 2
            lower_mask = timesteps_padded[:, None, None] < timestep[None] - t_eps
            lower_timestep = lower_mask * timesteps_padded[:, None, None]
            lower_timestep, _ = lower_timestep.max(dim=0)
            dt = (timestep - lower_timestep)[..., None]

        # Compute previous sample
        if stochastic_sampling:
            x0 = sample - timestep[..., None] * model_output
            next_timestep = timestep[..., None] - dt
            prev_sample = self.add_noise(x0, torch.randn_like(sample), next_timestep)
        else:
            prev_sample = sample - dt * model_output

        if not return_dict:
            return (prev_sample,)

        return RectifiedFlowSchedulerOutput(prev_sample=prev_sample)

    def add_noise(
        self,
        original_samples: torch.FloatTensor,
        noise: torch.FloatTensor,
        timesteps: torch.FloatTensor,
    ) -> torch.FloatTensor:
        sigmas = timesteps
        sigmas = append_dims(sigmas, original_samples.ndim)
        alphas = 1 - sigmas
        noisy_samples = alphas * original_samples + sigmas * noise
        return noisy_samples