gigaam_transformers.py

from typing import Dict, List, Optional, Union, Tuple

import numpy as np
import torch
import torch.nn as nn
import torchaudio
from .encoder import ConformerEncoder
from torch import Tensor
from torch.nn.utils.rnn import pad_sequence
from transformers import Wav2Vec2CTCTokenizer, Wav2Vec2Processor
from transformers.configuration_utils import PretrainedConfig
from transformers.feature_extraction_sequence_utils import \
    SequenceFeatureExtractor
from transformers.feature_extraction_utils import BatchFeature
from transformers.modeling_outputs import CausalLMOutput, Seq2SeqLMOutput
from transformers.modeling_utils import PreTrainedModel


class GigaAMCTC(nn.Module):
    """
    GigaAM-CTC model
    """

    def __init__(self, config_encoder, config_head):
        super().__init__()
        self.encoder = ConformerEncoder(**config_encoder)
        self.head = CTCHead(**config_head)

    def forward(self, input_features: Tensor, input_lengths: Tensor) -> Tensor:
        encoded, encoded_lengths = self.encoder(input_features, input_lengths)
        logits = self.head(encoded)
        return logits, encoded_lengths


class GigaAMRNNT(nn.Module):
    """
    GigaAM-RNNT model
    """

    def __init__(self, config_encoder, config_head):
        super().__init__()
        self.encoder = ConformerEncoder(**config_encoder)
        self.head = RNNTHead(**config_head)

    def forward(self, input_features: Tensor, input_lengths: Tensor, targets: Tensor, target_lengths: Tensor) -> Tensor:
        encoded, encoded_lengths = self.encoder(input_features, input_lengths)
        # During training, loss must be computed, so decoder forward is necessary
        decoder_out, target_lengths, states = self.head.decoder(targets=targets, target_length=target_lengths)
        joint = self.head.joint(encoder_outputs=encoded, decoder_outputs=decoder_out)
        # loss = self.loss(
        #     log_probs=joint, targets=targets, input_lengths=encoded_lengths, target_lengths=target_lengths
        # )
        
        return joint, encoded_lengths


class CTCHead(nn.Module):
    """
    CTC Head module for Connectionist Temporal Classification.
    """

    def __init__(self, feat_in: int, num_classes: int):
        super().__init__()
        self.decoder_layers = nn.Sequential(
            nn.Conv1d(feat_in, num_classes, kernel_size=1)
        )

    def forward(self, encoder_output: Tensor) -> Tensor:
        # B x C x T
        return self.decoder_layers(encoder_output)


class RNNTJoint(nn.Module):
    """
    RNN-Transducer Joint Network Module.
    This module combines the outputs of the encoder and the prediction network using
    a linear transformation followed by ReLU activation and another linear projection.
    """

    def __init__(
        self, enc_hidden: int, pred_hidden: int, joint_hidden: int, num_classes: int
    ):
        super().__init__()
        self.enc_hidden = enc_hidden
        self.pred_hidden = pred_hidden
        self.pred = nn.Linear(pred_hidden, joint_hidden)
        self.enc = nn.Linear(enc_hidden, joint_hidden)
        self.joint_net = nn.Sequential(nn.ReLU(), nn.Linear(joint_hidden, num_classes))

    def joint(self, encoder_out: Tensor, decoder_out: Tensor) -> Tensor:
        """
        Combine the encoder and prediction network outputs into a joint representation.
        """
        enc = self.enc(encoder_out).unsqueeze(2)
        pred = self.pred(decoder_out).unsqueeze(1)
        return self.joint_net(enc + pred)

    def input_example(self):
        device = next(self.parameters()).device
        enc = torch.zeros(1, self.enc_hidden, 1)
        dec = torch.zeros(1, self.pred_hidden, 1)
        return enc.float().to(device), dec.float().to(device)

    def input_names(self):
        return ["enc", "dec"]

    def output_names(self):
        return ["joint"]

    def forward(self, enc: Tensor, dec: Tensor) -> Tensor:
        return self.joint(enc.transpose(1, 2), dec.transpose(1, 2))


class RNNTDecoder(nn.Module):
    """
    RNN-Transducer Decoder Module.
    This module handles the prediction network part of the RNN-Transducer architecture.
    """

    def __init__(self, pred_hidden: int, pred_rnn_layers: int, num_classes: int):
        super().__init__()
        self.blank_id = num_classes - 1
        self.pred_hidden = pred_hidden
        self.embed = nn.Embedding(num_classes, pred_hidden, padding_idx=self.blank_id)
        self.lstm = nn.LSTM(pred_hidden, pred_hidden, pred_rnn_layers)

    def predict(
        self,
        x: Optional[Tensor],
        state: Optional[Tensor],
        batch_size: int = 1,
    ) -> Tuple[Tensor, Tensor]:
        """
        Make predictions based on the current input and previous states.
        If no input is provided, use zeros as the initial input.
        """
        if x is not None:
            emb: Tensor = self.embed(x)
        else:
            emb = torch.zeros(
                (batch_size, 1, self.pred_hidden), device=next(self.parameters()).device
            )
        g, hid = self.lstm(emb.transpose(0, 1), state)
        return g.transpose(0, 1), hid

    def input_example(self):
        device = next(self.parameters()).device
        label = torch.tensor([[0]]).to(device)
        hidden_h = torch.zeros(1, 1, self.pred_hidden).to(device)
        hidden_c = torch.zeros(1, 1, self.pred_hidden).to(device)
        return label, hidden_h, hidden_c

    def input_names(self):
        return ["x", "h", "c"]

    def output_names(self):
        return ["dec", "h", "c"]

    def forward(self, x: Tensor, h: Tensor, c: Tensor) -> Tuple[Tensor, Tensor, Tensor]:
        """
        ONNX-specific forward with x, state = (h, c) -> x, h, c.
        """
        emb = self.embed(x)
        g, (h, c) = self.lstm(emb.transpose(0, 1), (h, c))
        return g.transpose(0, 1), h, c


class RNNTHead(nn.Module):
    """
    RNN-Transducer Head Module.
    This module combines the decoder and joint network components of the RNN-Transducer architecture.
    """

    def __init__(self, decoder: Dict[str, int], joint: Dict[str, int]):
        super().__init__()
        self.decoder = RNNTDecoder(**decoder)
        self.joint = RNNTJoint(**joint)


class GigaAMFeatureExtractor(SequenceFeatureExtractor):
    """
    Feature extractor for GigaAM.
    """
    model_input_names = ["input_features"]

    def __init__(
        self,
        feature_size=64,
        sampling_rate=16000,
        padding_value=0.0,
        chunk_length=30.0,
        **kwargs,
    ):
        super().__init__(
            feature_size=feature_size,
            sampling_rate=sampling_rate,
            padding_value=padding_value,
            chunk_length=chunk_length,
            **kwargs,
        )
        self.hop_length = sampling_rate // 100
        self.n_samples = chunk_length * sampling_rate
        self.featurizer = torchaudio.transforms.MelSpectrogram(
                sample_rate=sampling_rate,
                n_fft=sampling_rate // 40,
                win_length=sampling_rate // 40,
                hop_length=self.hop_length,
                n_mels=feature_size,
            )

    def to_dict(self) -> Dict[str, Union[str, int, Dict]]:
        dictionary = super().to_dict()

        if "featurizer" in dictionary:
            del dictionary["featurizer"]
        dictionary["hop_length"] = self.hop_length
        dictionary["n_samples"] = self.n_samples
        return dictionary

    def out_len(self, input_lengths: Tensor) -> Tensor:
        """
        Calculates the output length after the feature extraction process.
        """
        return input_lengths.div(self.hop_length, rounding_mode="floor").add(1).long()

    def __call__(
        self,
        raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
        sampling_rate: Optional[int] = None,
        padding: str = "max_length",
        **kwargs,
    ):
        is_batched_numpy = (
            isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
        )
        if is_batched_numpy and len(raw_speech.shape) > 2:
            raise ValueError(
                f"Only mono-channel audio is supported for input to {self}"
            )
        is_batched = is_batched_numpy or (
            isinstance(raw_speech, (list, tuple))
            and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
        )

        if is_batched:
            raw_speech = [
                np.asarray([speech], dtype=np.float32).T for speech in raw_speech
            ]
        elif not is_batched and not isinstance(raw_speech, np.ndarray):
            raw_speech = np.asarray(raw_speech, dtype=np.float32)
        elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(
            np.float64
        ):
            raw_speech = raw_speech.astype(np.float32)

        # always return batch
        if not is_batched:
            raw_speech = [np.asarray([raw_speech]).T]

        input_lengths = torch.tensor([len(speech) for speech in raw_speech])

        batched_speech = BatchFeature({"input_features": raw_speech})

        padded_inputs = self.pad(
            batched_speech,
            padding=padding,
            max_length=self.n_samples,
            truncation=False,
            return_tensors="pt",
        )

        input_features = padded_inputs["input_features"].transpose(1, 2)
        input_features = self.featurizer(input_features).squeeze(1)
        input_features = torch.log(input_features.clamp_(1e-9, 1e9))
        input_lengths = self.out_len(input_lengths)

        return BatchFeature({"input_features": input_features, "input_lengths": input_lengths}, tensor_type="pt")


class GigaAMTokenizer(Wav2Vec2CTCTokenizer):
    """
    Char tokenizer for GigaAM model.
    """
    def __init__(
        self,
        vocab_file,
        unk_token="[BLANK]",
        pad_token="[BLANK]",
        bos_token=None,
        eos_token=None,
        word_delimiter_token=" ",
        **kwargs,
    ):
        super().__init__(
            vocab_file=vocab_file,
            unk_token=unk_token,
            pad_token=pad_token,
            bos_token=bos_token,
            eos_token=eos_token,
            word_delimiter_token=word_delimiter_token,
            **kwargs,
        )


class GigaAMProcessor(Wav2Vec2Processor):
    feature_extractor_class = "GigaAMFeatureExtractor"
    tokenizer_class = "GigaAMTokenizer"

    def __init__(self, feature_extractor, tokenizer):
        # super().__init__(feature_extractor, tokenizer)
        self.feature_extractor = feature_extractor
        self.tokenizer = tokenizer
        self.current_processor = self.feature_extractor
        self._in_target_context_manager = False

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
        feature_extractor = GigaAMFeatureExtractor.from_pretrained(pretrained_model_name_or_path, **kwargs)
        tokenizer = GigaAMTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs)

        return cls(feature_extractor=feature_extractor, tokenizer=tokenizer)


class GigaAMConfig(PretrainedConfig):
    model_type = "gigaam"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)


class GigaAMCTCHF(PreTrainedModel):
    """
    GigaAM-CTC model for transformers
    """
    config_class = GigaAMConfig
    base_model_prefix = "gigaamctc"
    main_input_name = "input_features"

    def __init__(self, config: GigaAMConfig):
        super().__init__(config)
        self.model = GigaAMCTC(config.encoder, config.head)

    def forward(self, input_features, input_lengths, labels=None, **kwargs):

        # B x C x T
        logits, encoded_lengths = self.model(input_features, input_lengths)
        # B x C x T -> B x T x C -> T x B x C
        log_probs = torch.log_softmax(
            logits.transpose(1, 2), dim=-1, dtype=torch.float32
        ).transpose(0, 1)

        loss = None
        if labels is not None:
            labels_mask = labels >= 0
            target_lengths = labels_mask.sum(-1)
            flattened_targets = labels.masked_select(labels_mask)

            loss = nn.functional.ctc_loss(
                log_probs,
                flattened_targets,
                encoded_lengths,
                target_lengths,
                blank=self.config.blank_id,
                zero_infinity=True,
            )

        return CausalLMOutput(loss=loss, logits=logits.transpose(1, 2))


class GigaAMRNNTHF(PreTrainedModel):
    """
    GigaAM-RNNT model for transformers
    """
    config_class = GigaAMConfig
    base_model_prefix = "gigaamrnnt"
    main_input_name = "input_features"

    def __init__(self, config: GigaAMConfig):
        super().__init__(config)
        self.model = GigaAMRNNT(config.encoder, config.head)

    def forward(self, input_features, input_lengths, labels=None, **kwargs):

        # B x C x T
        encoder_out, encoded_lengths = self.model.encoder(input_features, input_lengths)
        encoder_out = encoder_out.transpose(1, 2)
        batch_size = encoder_out.shape[0]

        loss = None
        if labels is not None:
            labels = labels.to(torch.int32)
            labels_mask = labels >= 0
            target_lengths = labels_mask.sum(-1).to(torch.int32)

            hidden_states = torch.zeros((self.config.head["decoder"]["pred_rnn_layers"], batch_size, self.model.head.decoder.pred_hidden), device=encoder_out.device)
            hidden_c = torch.zeros((self.config.head["decoder"]["pred_rnn_layers"], batch_size, self.model.head.decoder.pred_hidden), device=encoder_out.device)
            plus_one_dim = self.config.blank_id * torch.ones((batch_size, 1), dtype=torch.int32, device=encoder_out.device)
            labels[labels < 0] = self.config.blank_id
            
            decoder_out, h, c = self.model.head.decoder(torch.cat((plus_one_dim, labels), dim=1), hidden_states, hidden_c)

            joint = self.model.head.joint.joint(encoder_out, decoder_out)
            loss = torchaudio.functional.rnnt_loss(
                logits=joint,
                targets=labels,
                logit_lengths=encoded_lengths,
                target_lengths=target_lengths,
                blank=self.config.blank_id,
            )

        return Seq2SeqLMOutput(loss=loss, logits=encoder_out.transpose(1, 2))
    
    def _greedy_decode(self, x: Tensor, seqlen: Tensor) -> str:
        """
        Internal helper function for performing greedy decoding on a single sequence.
        """
        hyp: List[int] = []
        dec_state: Optional[Tensor] = None
        last_label: Optional[Tensor] = None
        for t in range(seqlen):
            f = x[t, :, :].unsqueeze(1)
            not_blank = True
            new_symbols = 0
            while not_blank and new_symbols < self.config.max_symbols:
                g, hidden = self.model.head.decoder.predict(last_label, dec_state)
                k = self.model.head.joint.joint(f, g)[0, 0, 0, :].argmax(0).item()
                if k == self.config.blank_id:
                    not_blank = False
                else:
                    hyp.append(k)
                    dec_state = hidden
                    last_label = torch.tensor([[hyp[-1]]]).to(x.device)
                    new_symbols += 1

        return torch.tensor(hyp, dtype=torch.int32)

    @torch.inference_mode()
    def generate(self, input_features: Tensor, input_lengths: Tensor, **kwargs) -> torch.Tensor:
        """
        Decode the output of an RNN-T model into a list of hypotheses.
        """
        encoder_out, encoded_lengths = self.model.encoder(input_features, input_lengths)
        encoder_out = encoder_out.transpose(1, 2)
        b = encoder_out.shape[0]
        preds = []
        for i in range(b):
            inseq = encoder_out[i, :, :].unsqueeze(1)
            preds.append(self._greedy_decode(inseq, encoded_lengths[i]))
        return pad_sequence(preds, batch_first=True, padding_value=self.config.blank_id)