Updates Round3

app.py

@@ -17,6 +17,7 @@ from pathlib import Path
 import time
 import typing as tp
 import warnings
+from tqdm import tqdm
 from audiocraft.models import MusicGen
 from audiocraft.data.audio import audio_write
 from audiocraft.data.audio_utils import apply_fade, apply_tafade, apply_splice_effect
@@ -48,6 +49,7 @@ os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:128,expandable_segmen
 # os.environ['USE_FLASH_ATTENTION'] = '1'
 # os.environ['XFORMERS_FORCE_DISABLE_TRITON']= '1'
 
+
 def interrupt_callback():
     return INTERRUPTED
 
@@ -162,7 +164,7 @@ def load_melody_filepath(melody_filepath, title, assigned_model):
 
     return  gr.update(value=melody_name), gr.update(maximum=MAX_PROMPT_INDEX, value=0), gr.update(value=assigned_model, interactive=True)
 
-def predict(model, text, melody_filepath, duration, dimension, topk, topp, temperature, cfg_coef, background, title, settings_font, settings_font_color, seed, overlap=1, prompt_index = 0, include_title = True, include_settings = True, harmony_only = False):
+def predict(model, text, melody_filepath, duration, dimension, topk, topp, temperature, cfg_coef, background, title, settings_font, settings_font_color, seed, overlap=1, prompt_index = 0, include_title = True, include_settings = True, harmony_only = False, profile = gr.OAuthProfile, progress=gr.Progress(track_tqdm=True)):
     global MODEL, INTERRUPTED, INTERRUPTING, MOVE_TO_CPU
     output_segments = None
     melody_name = "Not Used"
@@ -228,14 +230,16 @@ def predict(model, text, melody_filepath, duration, dimension, topk, topp, tempe
             cfg_coef=cfg_coef,
             duration=segment_duration,
             two_step_cfg=False,
+            extend_stride=10,
             rep_penalty=0.5
         )
+        MODEL.set_custom_progress_callback(gr.Progress(track_tqdm=True))
 
         try:
             if melody:
                 # return excess duration, load next model and continue in loop structure building up output_segments
                 if duration > MODEL.lm.cfg.dataset.segment_duration:
-                    output_segments, duration = generate_music_segments(text, melody, seed, MODEL, duration, overlap, MODEL.lm.cfg.dataset.segment_duration, prompt_index, harmony_only=False)
+                    output_segments, duration = generate_music_segments(text, melody, seed, MODEL, duration, overlap, MODEL.lm.cfg.dataset.segment_duration, prompt_index, harmony_only=False, progress=gr.Progress(track_tqdm=True))
                 else:
                     # pure original code
                     sr, melody = melody[0], torch.from_numpy(melody[1]).to(MODEL.device).float().t().unsqueeze(0)
@@ -247,20 +251,20 @@ def predict(model, text, melody_filepath, duration, dimension, topk, topp, tempe
                         descriptions=[text],
                         melody_wavs=melody,
                         melody_sample_rate=sr,
-                        progress=False
+                        progress=True, progress_callback=gr.Progress(track_tqdm=True)
                     )
                 # All output_segments are populated, so we can break the loop or set duration to 0
                 break
             else:
                 #output = MODEL.generate(descriptions=[text], progress=False)
                 if not output_segments:
-                    next_segment = MODEL.generate(descriptions=[text], progress=True)
+                    next_segment = MODEL.generate(descriptions=[text], progress=True, progress_callback=gr.Progress(track_tqdm=True))
                     duration -= segment_duration
                 else:
                     last_chunk = output_segments[-1][:, :, -overlap*MODEL.sample_rate:]
-                    next_segment = MODEL.generate_continuation(last_chunk, MODEL.sample_rate, descriptions=[text], progress=True)
+                    next_segment = MODEL.generate_continuation(last_chunk, MODEL.sample_rate, descriptions=[text], progress=True, progress_callback=gr.Progress(track_tqdm=True))
                     duration -= segment_duration - overlap
-                if next_segment != None:                
+                if next_segment != None:
                     output_segments.append(next_segment)
         except Exception as e:
             print(f"Error generating audio: {e}")
@@ -312,7 +316,7 @@ def predict(model, text, melody_filepath, duration, dimension, topk, topp, tempe
             return None, None, seed
         else:
             output = output.detach().cpu().float()[0]
-    profile: gr.OAuthProfile | None = None
+
     title_file_name = convert_title_to_filename(title)
     with NamedTemporaryFile("wb", suffix=".wav", delete=False, prefix = title_file_name) as file:
         video_description = f"{text}\n Duration: {str(initial_duration)} Dimension: {dimension}\n Top-k:{topk} Top-p:{topp}\n Randomness:{temperature}\n cfg:{cfg_coef} overlap: {overlap}\n Seed: {seed}\n Model: {model}\n Melody Condition:{melody_name}\n Sample Segment: {prompt_index}"
@@ -357,7 +361,7 @@ def predict(model, text, melody_filepath, duration, dimension, topk, topp, tempe
                 "background": background,
                 "include_title": include_title,
                 "include_settings": include_settings,
-                "profile": profile,
+                "profile": "Satoshi Nakamoto" if profile.value is None else profile.value.username,
                 "commit": commit_hash(),
                 "tag": git_tag(),
                 "version": gr.__version__,
@@ -396,11 +400,11 @@ def predict(model, text, melody_filepath, duration, dimension, topk, topp, tempe
 
         if waveform_video_path:
             modules.user_history.save_file(
-            profile=profile,
+            profile=profile.value,
             image=background,
-            audio=file,
+            audio=file.name,
             video=waveform_video_path,
-            label=text,
+            label=title,
             metadata=metadata,
         )
         
@@ -423,7 +427,7 @@ def ui(**kwargs):
             This is your private demo for [UnlimitedMusicGen](https://github.com/Oncorporation/audiocraft), a simple and controllable model for music generation
             presented at: ["Simple and Controllable Music Generation"](https://huggingface.co/papers/2306.05284)
             
-                Disclaimer: This won't run on CPU only. Clone this App and run on GPU instance!
+            Disclaimer: This won't run on CPU only. Clone this App and run on GPU instance!
                         
             Todo: Working on improved Interrupt.
             Theme Available at ["Surn/Beeuty"](https://huggingface.co/spaces/Surn/Beeuty)
@@ -482,12 +486,12 @@ def ui(**kwargs):
                     with gr.Column() as c:
                         output = gr.Video(label="Generated Music")
                         wave_file = gr.File(label=".wav file", elem_id="output_wavefile", interactive=True)
-                        seed_used = gr.Number(label='Seed used', value=-1, interactive=False)
+                        seed_used = gr.Number(label='Seed used', value=-1, interactive=False)                        
 
             radio.change(toggle_audio_src, radio, [melody_filepath], queue=False, show_progress=False)
             melody_filepath.change(load_melody_filepath, inputs=[melody_filepath, title, model], outputs=[title, prompt_index , model], api_name="melody_filepath_change", queue=False)
             reuse_seed.click(fn=lambda x: x, inputs=[seed_used], outputs=[seed], queue=False, api_name="reuse_seed")
-            submit.click(predict, inputs=[model, text,melody_filepath, duration, dimension, topk, topp, temperature, cfg_coef, background, title, settings_font, settings_font_color, seed, overlap, prompt_index, include_title, include_settings, harmony_only], outputs=[output, wave_file, seed_used], api_name="submit")
+            
             gr.Examples(
                 examples=[
                     [
@@ -524,10 +528,25 @@ def ui(**kwargs):
                 inputs=[text, melody_filepath, model, title],
                 outputs=[output]
             )
-            gr.HTML(value=versions_html(), visible=True, elem_id="versions")
+            
         with gr.Tab("User History") as history_tab:
             modules.user_history.render()
+        user_profile = gr.State(None)
             
+        with gr.Row("Versions") as versions_row:
+            gr.HTML(value=versions_html(), visible=True, elem_id="versions")
+
+        submit.click(
+            modules.user_history.get_profile,
+            inputs=[],
+            outputs=[user_profile],
+            queue=True,
+            api_name="submit"
+         ).then(
+             predict,
+             inputs=[model, text,melody_filepath, duration, dimension, topk, topp, temperature, cfg_coef, background, title, settings_font, settings_font_color, seed, overlap, prompt_index, include_title, include_settings, harmony_only, user_profile],
+             outputs=[output, wave_file, seed_used])
+
         # Show the interface
         launch_kwargs = {}
         share = kwargs.get('share', False)

audiocraft/__init__.py

@@ -7,4 +7,4 @@
 # flake8: noqa
 from . import data, modules, models
 
-__version__ = '1.4.Surn'
+__version__ = '1.2.Surn'

audiocraft/models/musicgen.py

@@ -15,6 +15,7 @@ import warnings
 
 import omegaconf
 import torch
+import gradio as gr
 
 from .encodec import CompressionModel
 from .lm import LMModel
@@ -67,7 +68,7 @@ class MusicGen:
         self.device = next(iter(lm.parameters())).device
         self.generation_params: dict = {}
         self.set_generation_params(duration=self.duration)  # 15 seconds by default
-        self._progress_callback: tp.Optional[tp.Callable[[int, int], None]] = None
+        self._progress_callback: tp.Union[tp.Callable[[int, int], None], gr.Progress] = None
         if self.device.type == 'cpu':
             self.autocast = TorchAutocast(enabled=False)
         else:
@@ -142,7 +143,7 @@ class MusicGen:
     def set_generation_params(self, use_sampling: bool = True, top_k: int = 250,
                               top_p: float = 0.0, temperature: float = 1.0,
                               duration: float = 30.0, cfg_coef: float = 3.0,
-                              two_step_cfg: bool = False, extend_stride: float = 18, rep_penalty: float = None):
+                              two_step_cfg: bool = False, extend_stride: float = 10, rep_penalty: float = None):
         """Set the generation parameters for MusicGen.
 
         Args:
@@ -173,12 +174,12 @@ class MusicGen:
             'two_step_cfg': two_step_cfg,
         }
 
-    def set_custom_progress_callback(self, progress_callback: tp.Optional[tp.Callable[[int, int], None]] = None):
+    def set_custom_progress_callback(self, progress_callback: tp.Union[tp.Callable[[int, int], None],gr.Progress] = None):
         """Override the default progress callback."""
         self._progress_callback = progress_callback
 
     def generate_unconditional(self, num_samples: int, progress: bool = False,
-                               return_tokens: bool = False) -> tp.Union[torch.Tensor,
+                               return_tokens: bool = False, progress_callback: gr.Progress = None) -> tp.Union[torch.Tensor, 
                                                                         tp.Tuple[torch.Tensor, torch.Tensor]]:
         """Generate samples in an unconditional manner.
 
@@ -194,7 +195,7 @@ class MusicGen:
             return self.generate_audio(tokens), tokens
         return self.generate_audio(tokens)
 
-    def generate(self, descriptions: tp.List[str], progress: bool = False, return_tokens: bool = False) \
+    def generate(self, descriptions: tp.List[str], progress: bool = False, return_tokens: bool = False, progress_callback: gr.Progress = None) \
             -> tp.Union[torch.Tensor, tp.Tuple[torch.Tensor, torch.Tensor]]:
         """Generate samples conditioned on text.
 
@@ -212,7 +213,7 @@ class MusicGen:
 
     def generate_with_chroma(self, descriptions: tp.List[str], melody_wavs: MelodyType,
                              melody_sample_rate: int, progress: bool = False,
-                             return_tokens: bool = False) -> tp.Union[torch.Tensor,
+                             return_tokens: bool = False, progress_callback=gr.Progress(track_tqdm=True)) -> tp.Union[torch.Tensor,
                                                                       tp.Tuple[torch.Tensor, torch.Tensor]]:
         """Generate samples conditioned on text and melody.
 
@@ -250,7 +251,7 @@ class MusicGen:
         return self.generate_audio(tokens)
 
     def generate_with_all(self, descriptions: tp.List[str], melody_wavs: MelodyType,
-                             sample_rate: int, progress: bool = False, prompt: tp.Optional[torch.Tensor] = None, return_tokens: bool = False) \
+                             sample_rate: int, progress: bool = False, prompt: tp.Optional[torch.Tensor] = None, return_tokens: bool = False, progress_callback: gr.Progress = None) \
             -> tp.Union[torch.Tensor, tp.Tuple[torch.Tensor, torch.Tensor]]:
         """Generate samples conditioned on text and melody and audio prompts.
         Args:
@@ -307,7 +308,7 @@ class MusicGen:
 
     def generate_continuation(self, prompt: torch.Tensor, prompt_sample_rate: int,
                               descriptions: tp.Optional[tp.List[tp.Optional[str]]] = None,
-                              progress: bool = False, return_tokens: bool = False) \
+                              progress: bool = False, return_tokens: bool = False, progress_callback: gr.Progress = None) \
             -> tp.Union[torch.Tensor, tp.Tuple[torch.Tensor, torch.Tensor]]:
         """Generate samples conditioned on audio prompts.
 
@@ -317,7 +318,8 @@ class MusicGen:
             prompt_sample_rate (int): Sampling rate of the given audio waveforms.
             descriptions (list of str, optional): A list of strings used as text conditioning. Defaults to None.
             progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
-            return_tokens (bool, optional): If True, also return the generated tokens. Defaults to False.
+            return_tokens (bool, optional): If True, also return the generated tokens. Defaults to False.\
+            This is truly a hack and does not follow the progression of conditioning melody or previously generated audio.
         """
         if prompt.dim() == 2:
             prompt = prompt[None]
@@ -338,7 +340,8 @@ class MusicGen:
             self,
             descriptions: tp.Sequence[tp.Optional[str]],
             prompt: tp.Optional[torch.Tensor],
-            melody_wavs: tp.Optional[MelodyList] = None,
+            melody_wavs: tp.Optional[MelodyList] = None, 
+            progress_callback: tp.Optional[gr.Progress] = None
     ) -> tp.Tuple[tp.List[ConditioningAttributes], tp.Optional[torch.Tensor]]:
         """Prepare model inputs.
 
@@ -392,7 +395,7 @@ class MusicGen:
         return attributes, prompt_tokens
 
     def _generate_tokens(self, attributes: tp.List[ConditioningAttributes],
-                         prompt_tokens: tp.Optional[torch.Tensor], progress: bool = False) -> torch.Tensor:
+                         prompt_tokens: tp.Optional[torch.Tensor], progress: bool = False, progress_callback: gr.Progress = None) -> torch.Tensor:
         """Generate discrete audio tokens given audio prompt and/or conditions.
 
         Args:
@@ -411,17 +414,19 @@ class MusicGen:
             if self._progress_callback is not None:
                 # Note that total_gen_len might be quite wrong depending on the
                 # codebook pattern used, but with delay it is almost accurate.
-                self._progress_callback(generated_tokens, total_gen_len)
-            else:
+                self._progress_callback((generated_tokens / total_gen_len), f"Generated {generated_tokens}/{total_gen_len} tokens")
+            if progress_callback is not None:
+                # Update Gradio progress bar
+                progress_callback((generated_tokens / total_gen_len), f"Generated {generated_tokens}/{total_gen_len} tokens")
+            if progress:
                 print(f'{generated_tokens: 6d} / {total_gen_len: 6d}', end='\r')
 
         if prompt_tokens is not None:
             assert max_prompt_len >= prompt_tokens.shape[-1], \
                 "Prompt is longer than audio to generate"
 
-        callback = None
-        if progress:
-            callback = _progress_callback
+        # callback = None
+        callback = _progress_callback
 
         if self.duration <= self.max_duration:
             # generate by sampling from LM, simple case.
@@ -481,7 +486,7 @@ class MusicGen:
 
         # generate audio
 
-    def generate_audio(self, gen_tokens: torch.Tensor):        
+    def generate_audio(self, gen_tokens: torch.Tensor):
         try:
             """Generate Audio from tokens"""
             assert gen_tokens.dim() == 3

audiocraft/utils/extend.py

@@ -12,6 +12,7 @@ import requests
 from io import BytesIO
 from huggingface_hub import hf_hub_download
 import librosa
+import gradio as gr
 
 
 INTERRUPTING = False
@@ -48,7 +49,7 @@ def separate_audio_segments(audio, segment_duration=30, overlap=1):
     print(f"separate_audio_segments: {len(segments)} segments of length {segment_samples // sr} seconds")
     return segments
 
-def generate_music_segments(text, melody, seed, MODEL, duration:int=10, overlap:int=1, segment_duration:int=30, prompt_index:int=0, harmony_only:bool= False):
+def generate_music_segments(text, melody, seed, MODEL, duration:int=10, overlap:int=1, segment_duration:int=30, prompt_index:int=0, harmony_only:bool= False, progress= gr.Progress(track_tqdm=True)):
     # generate audio segments
     melody_segments = separate_audio_segments(melody, segment_duration, 0) 
     

modules/user_history.py

@@ -14,9 +14,11 @@ Useful links:
 - README: https://huggingface.co/spaces/Wauplin/gradio-user-history/blob/main/README.md
 - Source file: https://huggingface.co/spaces/Wauplin/gradio-user-history/blob/main/user_history.py
 - Discussions: https://huggingface.co/spaces/Wauplin/gradio-user-history/discussions
+
+Update by Surn (Charles Fettinger)
 """
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 import json
 import os
@@ -39,6 +41,13 @@ from mutagen.mp3 import MP3, EasyMP3
 import torchaudio
 import subprocess
 
+user_profile = gr.State(None)
+
+def get_profile() -> gr.OAuthProfile | None:
+    global user_profile
+    """Return the user profile if logged in, None otherwise."""
+
+    return user_profile
 
 def setup(folder_path: str | Path | None = None) -> None:
     user_history = _UserHistory()
@@ -205,23 +214,30 @@ def save_file(
         image_path = _copy_image(image, dst_folder=user_history._user_images_path(username))
         image_path = _add_metadata(image_path, metadata)
 
+    video_path = None
     # Copy video to storage
     if video is not None:    
         video_path = _copy_file(video, dst_folder=user_history._user_file_path(username, "videos"))
         video_path = _add_metadata(video_path, metadata)
 
+    audio_path = None
     # Copy audio to storage
     if audio is not None:     
         audio_path = _copy_file(audio, dst_folder=user_history._user_file_path(username, "audios"))
         audio_path = _add_metadata(audio_path, metadata)
     
+    document_path = None
     # Copy document to storage
     if document is not None:     
         document_path = _copy_file(document, dst_folder=user_history._user_file_path(username, "documents"))
         document_path = _add_metadata(document_path, metadata)
 
+    
+    # If no image, video, audio or document => nothing to save
+    if image_path is None and video_path is None and audio_path is None and document_path is None:
+        return
     # Save Json file
-    data = {"image_path": str(image_path), "video_path": str(video_path), "audio_path": str(audio_path), "document_path": str(document_path), "label": label, "metadata": metadata}
+    data = {"image_path": str(image_path), "video_path": str(video_path), "audio_path": str(audio_path), "document_path": str(document_path), "label": _UserHistory._sanitize_for_json(label), "metadata": _UserHistory._sanitize_for_json(metadata)}
     with user_history._user_lock(username):
         with user_history._user_jsonl_path(username).open("a") as f:
             f.write(json.dumps(data) + "\n")
@@ -266,14 +282,34 @@ class _UserHistory(object):
         path.mkdir(parents=True, exist_ok=True)
         return path
    
+    @staticmethod
+    def _sanitize_for_json(obj: Any) -> Any:
+        """
+        Recursively convert non-serializable objects into their string representation.
+        """
+        if isinstance(obj, dict):
+            return {str(key): _UserHistory._sanitize_for_json(value) for key, value in obj.items()}
+        elif isinstance(obj, list):
+            return [_UserHistory._sanitize_for_json(item) for item in obj]
+        elif isinstance(obj, (str, int, float, bool)) or obj is None:
+            return obj
+        elif hasattr(obj, "isoformat"):
+            # For datetime objects and similar.
+            return obj.isoformat()
+        else:
+            return str(obj)
     
 
 def _fetch_user_history(profile: gr.OAuthProfile | None) -> List[Tuple[str, str]]:
     """Return saved history for that user, if it exists."""
     # Cannot load history for logged out users
+    global user_profile
     if profile is None:
+        user_profile = gr.State(None)
         return []
-    username = profile["preferred_username"]
+    username = str(profile["preferred_username"])
+    
+    user_profile = gr.State(profile)
 
     user_history = _UserHistory()
     if not user_history.initialized:
@@ -290,7 +326,7 @@ def _fetch_user_history(profile: gr.OAuthProfile | None) -> List[Tuple[str, str]
         images = []
         for line in jsonl_path.read_text().splitlines():
             data = json.loads(line)
-            images.append((data["path"], data["label"] or ""))
+            images.append((data["image_path"], data["label"] or ""))
         return list(reversed(images))
 
 

style_20250331.css

@@ -102,6 +102,10 @@ a {
     position: relative !important;
 }
 
+.gradio-container {
+    position: relative;
+}
+
 .gradio-container::before {
     content: ' ';
     display: block;
@@ -122,13 +126,15 @@ a {
 .gradio-container::after {
     content: '';
     position: absolute;
-    top: 0;
+    top: -5%;
     left: -60%; /* Start off-screen */
-    width: 30%;
-    height: 100%;
-    background: linear-gradient( 92deg, rgba(255, 255, 255, 0) 25%, rgba(255, 255, 255, 0.60) 50%, rgba(255, 255, 255, 0) 75% );
-    animation: shine 30s infinite;
-    opacity:0.35;
+    width: 100%;
+    height: calc(100% + 150px);
+    background: -webkit-linear-gradient(to top right, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0) 45%, rgba(255, 255, 255, 0.5) 48%, rgba(255, 255, 255, 0.8) 50%, rgba(255, 255, 255, 0.5) 52%, rgba(255, 255, 255, 0) 57%, rgba(255, 255, 255, 0) 100%);
+    animation: 15s infinite shine;
+    animation: shine 20s infinite;
+    opacity: 0.35;
+    z-index:2;
 }
 
 #component-0, #component-1 {
@@ -213,4 +219,18 @@ a {
     100% {
         left: 125%;
     }
+}
+
+@keyframes shinebg {
+    0% {
+        background-position: center, -100% 0;
+    }
+
+    20% {
+        background-position: center, 100% 0;
+    }
+
+    100% {
+        background-position: center, 125% 0;
+    }
 }