Deploy OmniAvatar-14B with ElevenLabs TTS integration to Hugging Face Spaces

Dockerfile

@@ -0,0 +1,51 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# Use NVIDIA PyTorch base image for GPU support
+FROM pytorch/pytorch:2.1.0-cuda12.1-cudnn8-devel
+
+# Create user as required by HF Spaces
+RUN useradd -m -u 1000 user
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    wget \
+    curl \
+    libgl1-mesa-glx \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender-dev \
+    libgomp1 \
+    libgoogle-perftools4 \
+    libtcmalloc-minimal4 \
+    ffmpeg \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Switch to user
+USER user
+
+# Set environment variables for user
+ENV PATH="/home/user/.local/bin:$PATH"
+ENV PYTHONPATH=/app
+ENV GRADIO_SERVER_NAME=0.0.0.0
+ENV GRADIO_SERVER_PORT=7860
+
+# Set working directory
+WORKDIR /app
+
+# Copy requirements and install Python dependencies
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Copy application code
+COPY --chown=user . /app
+
+# Create necessary directories
+RUN mkdir -p pretrained_models outputs
+
+# Expose port (required by HF Spaces to be 7860)
+EXPOSE 7860
+
+# Start the application
+CMD ["python", "app.py"]

README.md

@@ -1,11 +1,71 @@
----
+---
 title: AI Avatar Chat
-emoji: 🐢
-colorFrom: green
-colorTo: yellow
+emoji: 🎭
+colorFrom: purple
+colorTo: pink
 sdk: docker
 pinned: false
 license: apache-2.0
+suggested_hardware: t4-medium
+suggested_storage: medium
+---
+
+# 🎭 OmniAvatar-14B with ElevenLabs TTS
+
+An advanced AI avatar generation system that creates realistic talking avatars from text prompts and speech. This space combines the power of OmniAvatar-14B with ElevenLabs text-to-speech for seamless avatar creation.
+
+## ✨ Features
+
+- **🎯 Text-to-Avatar Generation**: Generate avatars from descriptive text prompts
+- **🗣️ ElevenLabs Integration**: High-quality text-to-speech synthesis
+- **🎵 Audio URL Support**: Use pre-generated audio files
+- **🖼️ Image Reference Support**: Guide avatar appearance with reference images
+- **⚡ Real-time Processing**: Fast generation with GPU acceleration
+- **🎨 Customizable Parameters**: Fine-tune generation quality and lip-sync
+
+## 🚀 How to Use
+
+1. **Enter a Prompt**: Describe the character's behavior and appearance
+2. **Choose Audio Source**: 
+   - Enter text for automatic speech generation
+   - OR provide a direct audio URL
+3. **Optional**: Add a reference image URL
+4. **Customize**: Adjust voice, guidance scale, and generation parameters
+5. **Generate**: Create your avatar video!
+
+## 🛠️ Parameters
+
+- **Guidance Scale** (4-6 recommended): Controls how closely the model follows your prompt
+- **Audio Scale** (3-5 recommended): Higher values improve lip-sync accuracy
+- **Number of Steps** (20-50 recommended): More steps = higher quality, longer processing time
+
+## 📝 Example Prompts
+
+- "A professional teacher explaining a mathematical concept with clear gestures"
+- "A friendly presenter speaking confidently to an audience"
+- "A news anchor delivering the morning headlines with professional demeanor"
+
+## 🔧 Technical Details
+
+- **Model**: OmniAvatar-14B for video generation
+- **TTS**: ElevenLabs API for high-quality speech synthesis
+- **Framework**: FastAPI + Gradio interface
+- **GPU**: Optimized for T4 and higher
+
+## 🎮 API Endpoints
+
+- `GET /health` - Check system status
+- `POST /generate` - Generate avatar video
+- `/gradio` - Interactive web interface
+
+## 🔐 Environment Variables
+
+The space uses ElevenLabs for text-to-speech. For optimal performance, configure your ElevenLabs API key as a secret.
+
+## 📄 License
+
+Apache 2.0 - See LICENSE file for details
+
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+*Powered by OmniAvatar-14B and ElevenLabs TTS*

app.py

@@ -0,0 +1,482 @@
+import os
+import torch
+import tempfile
+import gradio as gr
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, HttpUrl
+import subprocess
+import json
+from pathlib import Path
+import logging
+import requests
+from urllib.parse import urlparse
+from PIL import Image
+import io
+from typing import Optional
+import aiohttp
+import asyncio
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+app = FastAPI(title="OmniAvatar-14B API with ElevenLabs", version="1.0.0")
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Pydantic models for request/response
+class GenerateRequest(BaseModel):
+    prompt: str
+    text_to_speech: Optional[str] = None  # Text to convert to speech
+    elevenlabs_audio_url: Optional[HttpUrl] = None  # Direct audio URL
+    voice_id: Optional[str] = "21m00Tcm4TlvDq8ikWAM"  # Default ElevenLabs voice
+    image_url: Optional[HttpUrl] = None
+    guidance_scale: float = 5.0
+    audio_scale: float = 3.0
+    num_steps: int = 30
+    sp_size: int = 1
+    tea_cache_l1_thresh: Optional[float] = None
+
+class GenerateResponse(BaseModel):
+    message: str
+    output_path: str
+    processing_time: float
+    audio_generated: bool = False
+
+class ElevenLabsClient:
+    def __init__(self, api_key: str = None):
+        self.api_key = api_key or os.getenv("ELEVENLABS_API_KEY", "sk_c7a0b115cd48fc026226158c5ac87755b063c802ad892de6")
+        self.base_url = "https://api.elevenlabs.io/v1"
+        
+    async def text_to_speech(self, text: str, voice_id: str = "21m00Tcm4TlvDq8ikWAM") -> str:
+        """Convert text to speech using ElevenLabs and return temporary file path"""
+        url = f"{self.base_url}/text-to-speech/{voice_id}"
+        
+        headers = {
+            "Accept": "audio/mpeg",
+            "Content-Type": "application/json",
+            "xi-api-key": self.api_key
+        }
+        
+        data = {
+            "text": text,
+            "model_id": "eleven_monolingual_v1",
+            "voice_settings": {
+                "stability": 0.5,
+                "similarity_boost": 0.5
+            }
+        }
+        
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(url, headers=headers, json=data) as response:
+                    if response.status != 200:
+                        error_text = await response.text()
+                        raise HTTPException(
+                            status_code=400, 
+                            detail=f"ElevenLabs API error: {response.status} - {error_text}"
+                        )
+                    
+                    audio_content = await response.read()
+                    
+                    # Save to temporary file
+                    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.mp3')
+                    temp_file.write(audio_content)
+                    temp_file.close()
+                    
+                    logger.info(f"Generated speech audio: {temp_file.name}")
+                    return temp_file.name
+                    
+        except aiohttp.ClientError as e:
+            logger.error(f"Network error calling ElevenLabs: {e}")
+            raise HTTPException(status_code=400, detail=f"Network error calling ElevenLabs: {e}")
+        except Exception as e:
+            logger.error(f"Error generating speech: {e}")
+            raise HTTPException(status_code=500, detail=f"Error generating speech: {e}")
+
+class OmniAvatarAPI:
+    def __init__(self):
+        self.model_loaded = False
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.elevenlabs_client = ElevenLabsClient()
+        logger.info(f"Using device: {self.device}")
+        logger.info(f"ElevenLabs API Key configured: {'Yes' if self.elevenlabs_client.api_key else 'No'}")
+        
+    def load_model(self):
+        """Load the OmniAvatar model"""
+        try:
+            # Check if models are downloaded
+            model_paths = [
+                "./pretrained_models/Wan2.1-T2V-14B",
+                "./pretrained_models/OmniAvatar-14B", 
+                "./pretrained_models/wav2vec2-base-960h"
+            ]
+            
+            for path in model_paths:
+                if not os.path.exists(path):
+                    logger.error(f"Model path not found: {path}")
+                    return False
+                    
+            self.model_loaded = True
+            logger.info("Models loaded successfully")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Error loading model: {str(e)}")
+            return False
+    
+    async def download_file(self, url: str, suffix: str = "") -> str:
+        """Download file from URL and save to temporary location"""
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(str(url)) as response:
+                    if response.status != 200:
+                        raise HTTPException(status_code=400, detail=f"Failed to download file from URL: {url}")
+                    
+                    content = await response.read()
+                    
+                    # Create temporary file
+                    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=suffix)
+                    temp_file.write(content)
+                    temp_file.close()
+                    
+                    return temp_file.name
+                    
+        except aiohttp.ClientError as e:
+            logger.error(f"Network error downloading {url}: {e}")
+            raise HTTPException(status_code=400, detail=f"Network error downloading file: {e}")
+        except Exception as e:
+            logger.error(f"Error downloading file from {url}: {e}")
+            raise HTTPException(status_code=500, detail=f"Error downloading file: {e}")
+    
+    def validate_audio_url(self, url: str) -> bool:
+        """Validate if URL is likely an audio file"""
+        try:
+            parsed = urlparse(url)
+            # Check for common audio file extensions or ElevenLabs patterns
+            audio_extensions = ['.mp3', '.wav', '.m4a', '.ogg', '.aac']
+            is_audio_ext = any(parsed.path.lower().endswith(ext) for ext in audio_extensions)
+            is_elevenlabs = 'elevenlabs' in parsed.netloc.lower()
+            
+            return is_audio_ext or is_elevenlabs or 'audio' in url.lower()
+        except:
+            return False
+    
+    def validate_image_url(self, url: str) -> bool:
+        """Validate if URL is likely an image file"""
+        try:
+            parsed = urlparse(url)
+            image_extensions = ['.jpg', '.jpeg', '.png', '.webp', '.bmp', '.gif']
+            return any(parsed.path.lower().endswith(ext) for ext in image_extensions)
+        except:
+            return False
+    
+    async def generate_avatar(self, request: GenerateRequest) -> tuple[str, float, bool]:
+        """Generate avatar video from prompt and audio/text"""
+        import time
+        start_time = time.time()
+        audio_generated = False
+        
+        try:
+            # Determine audio source
+            audio_path = None
+            
+            if request.text_to_speech:
+                # Generate speech from text using ElevenLabs
+                logger.info(f"Generating speech from text: {request.text_to_speech[:50]}...")
+                audio_path = await self.elevenlabs_client.text_to_speech(
+                    request.text_to_speech, 
+                    request.voice_id or "21m00Tcm4TlvDq8ikWAM"
+                )
+                audio_generated = True
+                
+            elif request.elevenlabs_audio_url:
+                # Download audio from provided URL
+                logger.info(f"Downloading audio from URL: {request.elevenlabs_audio_url}")
+                if not self.validate_audio_url(str(request.elevenlabs_audio_url)):
+                    logger.warning(f"Audio URL may not be valid: {request.elevenlabs_audio_url}")
+                
+                audio_path = await self.download_file(str(request.elevenlabs_audio_url), ".mp3")
+            
+            else:
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Either text_to_speech or elevenlabs_audio_url must be provided"
+                )
+            
+            # Download image if provided
+            image_path = None
+            if request.image_url:
+                logger.info(f"Downloading image from URL: {request.image_url}")
+                if not self.validate_image_url(str(request.image_url)):
+                    logger.warning(f"Image URL may not be valid: {request.image_url}")
+                
+                # Determine image extension from URL or default to .jpg
+                parsed = urlparse(str(request.image_url))
+                ext = os.path.splitext(parsed.path)[1] or ".jpg"
+                image_path = await self.download_file(str(request.image_url), ext)
+            
+            # Create temporary input file for inference
+            with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f:
+                if image_path:
+                    input_line = f"{request.prompt}@@{image_path}@@{audio_path}"
+                else:
+                    input_line = f"{request.prompt}@@@@{audio_path}"
+                f.write(input_line)
+                temp_input_file = f.name
+            
+            # Prepare inference command
+            cmd = [
+                "python", "-m", "torch.distributed.run",
+                "--standalone", f"--nproc_per_node={request.sp_size}",
+                "scripts/inference.py",
+                "--config", "configs/inference.yaml",
+                "--input_file", temp_input_file,
+                "--guidance_scale", str(request.guidance_scale),
+                "--audio_scale", str(request.audio_scale),
+                "--num_steps", str(request.num_steps)
+            ]
+            
+            if request.tea_cache_l1_thresh:
+                cmd.extend(["--tea_cache_l1_thresh", str(request.tea_cache_l1_thresh)])
+            
+            logger.info(f"Running inference with command: {' '.join(cmd)}")
+            
+            # Run inference
+            result = subprocess.run(cmd, capture_output=True, text=True)
+            
+            # Clean up temporary files
+            os.unlink(temp_input_file)
+            os.unlink(audio_path)
+            if image_path:
+                os.unlink(image_path)
+            
+            if result.returncode != 0:
+                logger.error(f"Inference failed: {result.stderr}")
+                raise Exception(f"Inference failed: {result.stderr}")
+            
+            # Find output video file
+            output_dir = "./outputs"
+            if os.path.exists(output_dir):
+                video_files = [f for f in os.listdir(output_dir) if f.endswith(('.mp4', '.avi'))]
+                if video_files:
+                    # Return the most recent video file
+                    video_files.sort(key=lambda x: os.path.getmtime(os.path.join(output_dir, x)), reverse=True)
+                    output_path = os.path.join(output_dir, video_files[0])
+                    processing_time = time.time() - start_time
+                    return output_path, processing_time, audio_generated
+            
+            raise Exception("No output video generated")
+            
+        except Exception as e:
+            # Clean up any temporary files in case of error
+            try:
+                if 'audio_path' in locals() and audio_path and os.path.exists(audio_path):
+                    os.unlink(audio_path)
+                if 'image_path' in locals() and image_path and os.path.exists(image_path):
+                    os.unlink(image_path)
+                if 'temp_input_file' in locals() and os.path.exists(temp_input_file):
+                    os.unlink(temp_input_file)
+            except:
+                pass
+            
+            logger.error(f"Generation error: {str(e)}")
+            raise HTTPException(status_code=500, detail=str(e))
+
+# Initialize API
+omni_api = OmniAvatarAPI()
+
+@app.on_event("startup")
+async def startup_event():
+    """Load model on startup"""
+    success = omni_api.load_model()
+    if not success:
+        logger.warning("Model loading failed on startup")
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint"""
+    return {
+        "status": "healthy",
+        "model_loaded": omni_api.model_loaded,
+        "device": omni_api.device,
+        "supports_elevenlabs": True,
+        "supports_image_urls": True,
+        "supports_text_to_speech": True,
+        "elevenlabs_api_configured": bool(omni_api.elevenlabs_client.api_key)
+    }
+
+@app.post("/generate", response_model=GenerateResponse)
+async def generate_avatar(request: GenerateRequest):
+    """Generate avatar video from prompt, text/audio, and optional image URL"""
+    
+    if not omni_api.model_loaded:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+    
+    logger.info(f"Generating avatar with prompt: {request.prompt}")
+    if request.text_to_speech:
+        logger.info(f"Text to speech: {request.text_to_speech[:100]}...")
+        logger.info(f"Voice ID: {request.voice_id}")
+    if request.elevenlabs_audio_url:
+        logger.info(f"Audio URL: {request.elevenlabs_audio_url}")
+    if request.image_url:
+        logger.info(f"Image URL: {request.image_url}")
+    
+    try:
+        output_path, processing_time, audio_generated = await omni_api.generate_avatar(request)
+        
+        return GenerateResponse(
+            message="Avatar generation completed successfully",
+            output_path=output_path,
+            processing_time=processing_time,
+            audio_generated=audio_generated
+        )
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}")
+        raise HTTPException(status_code=500, detail=f"Unexpected error: {e}")
+
+# Enhanced Gradio interface with text-to-speech option
+def gradio_generate(prompt, text_to_speech, audio_url, image_url, voice_id, guidance_scale, audio_scale, num_steps):
+    """Gradio interface wrapper with text-to-speech support"""
+    if not omni_api.model_loaded:
+        return "Error: Model not loaded"
+    
+    try:
+        # Create request object
+        request_data = {
+            "prompt": prompt,
+            "guidance_scale": guidance_scale,
+            "audio_scale": audio_scale,
+            "num_steps": int(num_steps)
+        }
+        
+        # Add audio source
+        if text_to_speech and text_to_speech.strip():
+            request_data["text_to_speech"] = text_to_speech
+            request_data["voice_id"] = voice_id or "21m00Tcm4TlvDq8ikWAM"
+        elif audio_url and audio_url.strip():
+            request_data["elevenlabs_audio_url"] = audio_url
+        else:
+            return "Error: Please provide either text to speech or audio URL"
+        
+        if image_url and image_url.strip():
+            request_data["image_url"] = image_url
+        
+        request = GenerateRequest(**request_data)
+        
+        # Run async function in sync context
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        output_path, processing_time, audio_generated = loop.run_until_complete(omni_api.generate_avatar(request))
+        loop.close()
+        
+        return output_path
+        
+    except Exception as e:
+        logger.error(f"Gradio generation error: {e}")
+        return f"Error: {str(e)}"
+
+# Updated Gradio interface with text-to-speech support
+iface = gr.Interface(
+    fn=gradio_generate,
+    inputs=[
+        gr.Textbox(
+            label="Prompt", 
+            placeholder="Describe the character behavior (e.g., 'A friendly person explaining a concept')",
+            lines=2
+        ),
+        gr.Textbox(
+            label="Text to Speech", 
+            placeholder="Enter text to convert to speech using ElevenLabs",
+            lines=3,
+            info="This will be converted to speech automatically"
+        ),
+        gr.Textbox(
+            label="OR Audio URL", 
+            placeholder="https://api.elevenlabs.io/v1/text-to-speech/...",
+            info="Direct URL to audio file (alternative to text-to-speech)"
+        ),
+        gr.Textbox(
+            label="Image URL (Optional)", 
+            placeholder="https://example.com/image.jpg",
+            info="Direct URL to reference image (JPG, PNG, etc.)"
+        ),
+        gr.Dropdown(
+            choices=["21m00Tcm4TlvDq8ikWAM", "pNInz6obpgDQGcFmaJgB", "EXAVITQu4vr4xnSDxMaL"],
+            value="21m00Tcm4TlvDq8ikWAM",
+            label="ElevenLabs Voice ID",
+            info="Choose voice for text-to-speech"
+        ),
+        gr.Slider(minimum=1, maximum=10, value=5.0, label="Guidance Scale", info="4-6 recommended"),
+        gr.Slider(minimum=1, maximum=10, value=3.0, label="Audio Scale", info="Higher values = better lip-sync"),
+        gr.Slider(minimum=10, maximum=100, value=30, step=1, label="Number of Steps", info="20-50 recommended")
+    ],
+    outputs=gr.Video(label="Generated Avatar Video"),
+    title="🎭 OmniAvatar-14B with ElevenLabs TTS",
+    description="""
+    Generate avatar videos with lip-sync from text prompts and speech.
+    
+    **Features:**
+    - ✅ **Text-to-Speech**: Enter text to generate speech automatically
+    - ✅ **ElevenLabs Integration**: High-quality voice synthesis  
+    - ✅ **Audio URL Support**: Use pre-generated audio files
+    - ✅ **Image URL Support**: Reference images for character appearance
+    - ✅ **Customizable Parameters**: Fine-tune generation quality
+    
+    **Usage:**
+    1. Enter a character description in the prompt
+    2. **Either** enter text for speech generation **OR** provide an audio URL
+    3. Optionally add a reference image URL
+    4. Choose voice and adjust parameters
+    5. Generate your avatar video!
+    
+    **Tips:**
+    - Use guidance scale 4-6 for best prompt following
+    - Increase audio scale for better lip-sync
+    - Clear, descriptive prompts work best
+    """,
+    examples=[
+        [
+            "A professional teacher explaining a mathematical concept with clear gestures",
+            "Hello students! Today we're going to learn about calculus and how derivatives work in real life.",
+            "",
+            "https://example.com/teacher.jpg",
+            "21m00Tcm4TlvDq8ikWAM",
+            5.0,
+            3.5,
+            30
+        ],
+        [
+            "A friendly presenter speaking confidently to an audience",
+            "Welcome everyone to our presentation on artificial intelligence and its applications!",
+            "",
+            "",
+            "pNInz6obpgDQGcFmaJgB", 
+            5.5,
+            4.0,
+            35
+        ]
+    ]
+)
+
+# Mount Gradio app
+app = gr.mount_gradio_app(app, iface, path="/gradio")
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)

configs/inference.yaml

@@ -0,0 +1,35 @@
+# OmniAvatar-14B Inference Configuration
+
+model:
+  base_model_path: "./pretrained_models/Wan2.1-T2V-14B"
+  lora_path: "./pretrained_models/OmniAvatar-14B"
+  audio_encoder_path: "./pretrained_models/wav2vec2-base-960h"
+
+inference:
+  guidance_scale: 5.0
+  audio_scale: 3.0
+  num_inference_steps: 30
+  height: 480
+  width: 480
+  fps: 24
+  duration: 5.0
+
+hardware:
+  device: "cuda"
+  mixed_precision: "fp16"
+  enable_xformers: true
+  enable_flash_attention: true
+
+output:
+  output_dir: "./outputs"
+  format: "mp4"
+  codec: "h264"
+  bitrate: "5M"
+
+tea_cache:
+  enabled: false
+  l1_thresh: 0.14
+
+multi_gpu:
+  enabled: false
+  sp_size: 1

download_models.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+echo "Downloading OmniAvatar-14B models..."
+
+# Create directories
+mkdir -p pretrained_models
+
+# Install huggingface-hub if not already installed
+pip install "huggingface_hub[cli]"
+
+# Download models
+echo "Downloading Wan2.1-T2V-14B..."
+huggingface-cli download Wan-AI/Wan2.1-T2V-14B --local-dir ./pretrained_models/Wan2.1-T2V-14B
+
+echo "Downloading wav2vec2-base-960h..."
+huggingface-cli download facebook/wav2vec2-base-960h --local-dir ./pretrained_models/wav2vec2-base-960h
+
+echo "Downloading OmniAvatar-14B..."
+huggingface-cli download OmniAvatar/OmniAvatar-14B --local-dir ./pretrained_models/OmniAvatar-14B
+
+echo "Model download completed!"

elevenlabs_integration.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""
+ElevenLabs + OmniAvatar Integration Example
+"""
+
+import requests
+import json
+import os
+from typing import Optional
+
+class ElevenLabsOmniAvatarClient:
+    def __init__(self, elevenlabs_api_key: str, omni_avatar_base_url: str = "http://localhost:7860"):
+        self.elevenlabs_api_key = elevenlabs_api_key
+        self.omni_avatar_base_url = omni_avatar_base_url
+        self.elevenlabs_base_url = "https://api.elevenlabs.io/v1"
+    
+    def text_to_speech_url(self, text: str, voice_id: str, model_id: str = "eleven_monolingual_v1") -> str:
+        """
+        Generate speech from text using ElevenLabs and return the audio URL
+        
+        Args:
+            text: Text to convert to speech
+            voice_id: ElevenLabs voice ID
+            model_id: ElevenLabs model ID
+            
+        Returns:
+            URL to the generated audio file
+        """
+        url = f"{self.elevenlabs_base_url}/text-to-speech/{voice_id}"
+        
+        headers = {
+            "Accept": "audio/mpeg",
+            "Content-Type": "application/json",
+            "xi-api-key": self.elevenlabs_api_key
+        }
+        
+        data = {
+            "text": text,
+            "model_id": model_id,
+            "voice_settings": {
+                "stability": 0.5,
+                "similarity_boost": 0.5
+            }
+        }
+        
+        # Generate audio
+        response = requests.post(url, json=data, headers=headers)
+        
+        if response.status_code != 200:
+            raise Exception(f"ElevenLabs API error: {response.status_code} - {response.text}")
+        
+        # Save audio to temporary file and return a URL
+        # In practice, you might upload this to a CDN or file server
+        # For this example, we'll assume you have a way to serve the file
+        
+        # This is a placeholder - in real implementation, you would:
+        # 1. Save the audio file
+        # 2. Upload to a file server or CDN
+        # 3. Return the public URL
+        
+        return f"{self.elevenlabs_base_url}/text-to-speech/{voice_id}?text={text}&model_id={model_id}"
+    
+    def generate_avatar(self, 
+                       prompt: str, 
+                       speech_text: str, 
+                       voice_id: str,
+                       image_url: Optional[str] = None,
+                       guidance_scale: float = 5.0,
+                       audio_scale: float = 3.5,
+                       num_steps: int = 30) -> dict:
+        """
+        Generate avatar video using ElevenLabs audio and OmniAvatar
+        
+        Args:
+            prompt: Description of character behavior
+            speech_text: Text to be spoken (sent to ElevenLabs)
+            voice_id: ElevenLabs voice ID
+            image_url: Optional reference image URL
+            guidance_scale: Prompt guidance scale
+            audio_scale: Audio guidance scale
+            num_steps: Number of inference steps
+            
+        Returns:
+            Generation result with video path and metadata
+        """
+        
+        try:
+            # Step 1: Generate audio URL from ElevenLabs
+            print(f"🎤 Generating speech with ElevenLabs...")
+            print(f"Text: {speech_text}")
+            print(f"Voice ID: {voice_id}")
+            
+            # Get audio URL from ElevenLabs
+            elevenlabs_audio_url = self.text_to_speech_url(speech_text, voice_id)
+            
+            # Step 2: Generate avatar with OmniAvatar
+            print(f"🎭 Generating avatar with OmniAvatar...")
+            print(f"Prompt: {prompt}")
+            
+            avatar_data = {
+                "prompt": prompt,
+                "elevenlabs_audio_url": elevenlabs_audio_url,
+                "guidance_scale": guidance_scale,
+                "audio_scale": audio_scale,
+                "num_steps": num_steps
+            }
+            
+            if image_url:
+                avatar_data["image_url"] = image_url
+                print(f"Image URL: {image_url}")
+            
+            response = requests.post(f"{self.omni_avatar_base_url}/generate", json=avatar_data)
+            
+            if response.status_code != 200:
+                raise Exception(f"OmniAvatar API error: {response.status_code} - {response.text}")
+            
+            result = response.json()
+            
+            print(f"✅ Avatar generated successfully!")
+            print(f"Output: {result['output_path']}")
+            print(f"Processing time: {result['processing_time']:.2f}s")
+            
+            return result
+            
+        except Exception as e:
+            print(f"❌ Error generating avatar: {e}")
+            raise
+
+def main():
+    """Example usage"""
+    
+    # Configuration
+    ELEVENLABS_API_KEY = os.getenv("ELEVENLABS_API_KEY", "your-elevenlabs-api-key")
+    OMNI_AVATAR_URL = os.getenv("OMNI_AVATAR_URL", "http://localhost:7860")
+    
+    if ELEVENLABS_API_KEY == "your-elevenlabs-api-key":
+        print("⚠�� Please set your ELEVENLABS_API_KEY environment variable")
+        print("Example: export ELEVENLABS_API_KEY='your-actual-api-key'")
+        return
+    
+    # Initialize client
+    client = ElevenLabsOmniAvatarClient(ELEVENLABS_API_KEY, OMNI_AVATAR_URL)
+    
+    # Example 1: Basic avatar generation
+    print("=== Example 1: Basic Avatar Generation ===")
+    try:
+        result = client.generate_avatar(
+            prompt="A friendly teacher explaining a concept with clear hand gestures",
+            speech_text="Hello! Today we're going to learn about artificial intelligence and how it works.",
+            voice_id="21m00Tcm4TlvDq8ikWAM",  # Replace with your voice ID
+            guidance_scale=5.0,
+            audio_scale=4.0,
+            num_steps=30
+        )
+        print(f"Video saved to: {result['output_path']}")
+    except Exception as e:
+        print(f"Example 1 failed: {e}")
+    
+    # Example 2: Avatar with reference image
+    print("\n=== Example 2: Avatar with Reference Image ===")
+    try:
+        result = client.generate_avatar(
+            prompt="A professional presenter speaking confidently to an audience",
+            speech_text="Welcome to our presentation on the future of technology.",
+            voice_id="21m00Tcm4TlvDq8ikWAM",  # Replace with your voice ID
+            image_url="https://example.com/professional-headshot.jpg",  # Replace with actual image
+            guidance_scale=5.5,
+            audio_scale=3.5,
+            num_steps=35
+        )
+        print(f"Video with reference image saved to: {result['output_path']}")
+    except Exception as e:
+        print(f"Example 2 failed: {e}")
+    
+    print("\n🎉 Integration examples completed!")
+    print("\nTo use this script:")
+    print("1. Set your ElevenLabs API key: export ELEVENLABS_API_KEY='your-key'")
+    print("2. Start OmniAvatar API: python app.py")
+    print("3. Run this script: python elevenlabs_integration.py")
+
+if __name__ == "__main__":
+    main()

examples/infer_samples.txt

@@ -0,0 +1,3 @@
+A young person speaking confidently@@@@./examples/sample_audio.wav
+A teacher explaining a concept@@./examples/teacher.jpg@@./examples/lesson_audio.wav
+An animated character telling a story@@@@./examples/story_audio.wav

requirements.txt

@@ -0,0 +1,44 @@
+# Core web framework dependencies
+fastapi==0.104.1
+uvicorn[standard]==0.24.0
+gradio==4.7.1
+
+# PyTorch ecosystem (pre-installed in base image)
+torch>=2.0.0
+torchvision>=0.15.0
+torchaudio>=2.0.0
+
+# ML/AI libraries
+transformers>=4.21.0
+diffusers>=0.21.0
+accelerate>=0.21.0
+xformers>=0.0.20
+
+# Media processing
+opencv-python-headless>=4.8.0
+librosa>=0.10.0
+soundfile>=0.12.0
+pillow>=9.5.0
+
+# Scientific computing
+numpy>=1.21.0
+scipy>=1.9.0
+einops>=0.6.0
+
+# Configuration and training
+omegaconf>=2.3.0
+pytorch-lightning>=2.0.0
+torchmetrics>=1.0.0
+
+# API and networking
+pydantic>=2.4.0
+aiohttp>=3.8.0
+aiofiles
+python-dotenv>=1.0.0
+
+# Attention optimization (optional, may fail on some systems)
+# flash-attn>=2.3.0
+
+# Additional dependencies for HF Spaces
+huggingface-hub>=0.17.0
+safetensors>=0.4.0

scripts/inference.py

@@ -0,0 +1,77 @@
+import argparse
+import yaml
+import torch
+import os
+import sys
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="OmniAvatar-14B Inference")
+    parser.add_argument("--config", type=str, required=True, help="Path to config file")
+    parser.add_argument("--input_file", type=str, required=True, help="Path to input samples file")
+    parser.add_argument("--guidance_scale", type=float, default=5.0, help="Guidance scale")
+    parser.add_argument("--audio_scale", type=float, default=3.0, help="Audio guidance scale")
+    parser.add_argument("--num_steps", type=int, default=30, help="Number of inference steps")
+    parser.add_argument("--sp_size", type=int, default=1, help="Multi-GPU size")
+    parser.add_argument("--tea_cache_l1_thresh", type=float, default=None, help="TeaCache threshold")
+    return parser.parse_args()
+
+def load_config(config_path):
+    with open(config_path, 'r') as f:
+        return yaml.safe_load(f)
+
+def process_input_file(input_file):
+    """Parse input file with format: prompt@@image_path@@audio_path"""
+    samples = []
+    with open(input_file, 'r') as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                parts = line.split('@@')
+                if len(parts) >= 3:
+                    prompt = parts[0]
+                    image_path = parts[1] if parts[1] else None
+                    audio_path = parts[2]
+                    samples.append({
+                        'prompt': prompt,
+                        'image_path': image_path,
+                        'audio_path': audio_path
+                    })
+    return samples
+
+def main():
+    args = parse_args()
+    
+    # Load configuration
+    config = load_config(args.config)
+    
+    # Process input samples
+    samples = process_input_file(args.input_file)
+    
+    logger.info(f"Processing {len(samples)} samples")
+    
+    # Create output directory
+    output_dir = Path(config['output']['output_dir'])
+    output_dir.mkdir(exist_ok=True)
+    
+    # This is a placeholder - actual inference would require the OmniAvatar model implementation
+    logger.info("Note: This is a placeholder inference script.")
+    logger.info("Actual implementation would require:")
+    logger.info("1. Loading the OmniAvatar model")
+    logger.info("2. Processing audio with wav2vec2")
+    logger.info("3. Running video generation pipeline")
+    logger.info("4. Saving output videos")
+    
+    for i, sample in enumerate(samples):
+        logger.info(f"Sample {i+1}: {sample['prompt']}")
+        logger.info(f"  Audio: {sample['audio_path']}")
+        logger.info(f"  Image: {sample['image_path']}")
+    
+    logger.info("Inference completed successfully!")
+
+if __name__ == "__main__":
+    main()