File size: 13,841 Bytes
729a510
37369e0
729a510
7fb6157
 
729a510
7fb6157
729a510
7fb6157
 
e7d2d44
c09190f
 
729a510
c09190f
 
6e97b7b
391222d
6c0447a
 
 
7c7eec0
 
391222d
c09190f
0aab8e8
729a510
9129aed
 
 
729a510
7fb6157
729a510
 
7fb6157
729a510
6c0447a
734997b
 
 
 
 
 
3f72ed6
 
7893b18
734997b
 
 
 
ca0138d
729a510
9da3dd1
729a510
38d399f
30800e2
c90f947
 
daa876c
38d399f
5431daf
bdf739a
 
 
 
3622a55
30800e2
729a510
bdf739a
5ce2521
 
bdf739a
 
 
 
83bec74
ff98c16
ca0138d
729a510
 
6c0447a
5ce2521
729a510
 
 
 
 
b86dcb7
729a510
 
 
 
 
 
 
 
ff98c16
729a510
 
 
5ce2521
729a510
 
 
 
 
b0f9db1
729a510
c90f947
729a510
 
c90f947
729a510
 
4d345df
33982b5
ab5b2ce
df19171
c90f947
 
 
a779e8d
6368dcf
c90f947
4d345df
ebaa2ef
4d345df
c90f947
6368dcf
 
 
 
ab5b2ce
 
6368dcf
ee50cff
c90f947
729a510
6eac10a
729a510
c90f947
 
 
 
 
 
 
070bf87
c90f947
 
 
070bf87
c90f947
 
 
070bf87
c90f947
 
 
070bf87
c90f947
 
 
070bf87
c90f947
 
 
070bf87
c90f947
 
 
070bf87
c90f947
 
 
2bbe8ed
d221946
c90f947
 
2bbe8ed
c90f947
 
 
 
 
 
 
 
 
 
 
 
ebaa2ef
c90f947
 
f4c6646
c90f947
 
1bd1938
 
7fb6157
 
 
 
 
 
 
 
 
1bd1938
7fb6157
 
 
 
1019525
 
cd6f6d9
7fb6157
 
 
 
 
 
 
 
 
9129aed
7fb6157
1d45d8a
0aaa7a9
7fb6157
 
 
 
 
 
 
c09190f
 
f4c6646
c09190f
 
7fb6157
391222d
a779e8d
01d2cdf
 
 
 
 
 
 
a779e8d
6e97b7b
 
 
 
 
 
7401c2d
6e97b7b
a779e8d
0ac1d72
 
 
 
 
c5bd206
0ac1d72
 
064b63f
439e8e0
03d90ab
439e8e0
 
064b63f
 
 
 
 
 
559bd0e
0ac1d72
 
 
03e0e2d
7fb6157
c819c3d
7fb6157
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ade087a
e5b0363
f1891cd
2500455
ab5b2ce
34eddaa
e5b0363
 
 
 
faf112e
92ac6d6
5ab821b
92a21ac
c819c3d
 
c90f947
c819c3d
e0dcf65
0ac1d72
 
c819c3d
ab5b2ce
7fb6157
391222d
ebaa2ef
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
import gradio as gr
import openai
import numpy as np
import time
import base64
import ffmpeg
from sentence_transformers import SentenceTransformer
from audio2numpy import open_audio
import httpx
import json
import os
import requests
import urllib
import pydub
from os import path
from pydub import AudioSegment
import re

MUBERT_LICENSE = os.environ.get('MUBERT_LICENSE')
MUBERT_TOKEN = os.environ.get('MUBERT_TOKEN')

#img_to_text = gr.Blocks.load(name="spaces/pharma/CLIP-Interrogator")
img_to_text = gr.Blocks.load(name="spaces/fffiloni/CLIP-Interrogator-2")

from share_btn import community_icon_html, loading_icon_html, share_js
from utils import get_tags_for_prompts, get_mubert_tags_embeddings

minilm = SentenceTransformer('all-MiniLM-L6-v2')
mubert_tags_embeddings = get_mubert_tags_embeddings(minilm)

##β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”

MUBERT_LICENSE = os.environ.get('MUBERT_LICENSE')
MUBERT_TOKEN = os.environ.get('MUBERT_TOKEN')

##β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”
def get_pat_token():
    r = httpx.post('https://api-b2b.mubert.com/v2/GetServiceAccess',
                   json={
                       "method": "GetServiceAccess",
                       "params": {
                           "email":"[email protected]",
                           "phone":"+11234567890",
                           "license": MUBERT_LICENSE,
                           "token": MUBERT_TOKEN,
                           
                       }
                   })

    rdata = json.loads(r.text)
    assert rdata['status'] == 1, "probably incorrect e-mail"
    pat = rdata['data']['pat']
    #print(f"pat: {pat}")
    return pat

def get_music(pat, prompt, track_duration, gen_intensity, gen_mode):

    if len(prompt) > 200:
        prompt = prompt[:200]
    
    r = httpx.post('https://api-b2b.mubert.com/v2/TTMRecordTrack',
                   json={
                       "method": "TTMRecordTrack",
                       "params":
                           {
                                "text": prompt,
                                "pat": pat,
                                "mode":gen_mode,
                                "duration":track_duration, 
                                "intensity": gen_intensity,
                                "format": "wav"
                           }
    })

    rdata = json.loads(r.text)
    
    #print(f"rdata: {rdata}")
    assert rdata['status'] == 1, rdata['error']['text']
    track = rdata['data']['tasks'][0]['download_link']
    print(track)
    
    local_file_path = "sample.wav"

    # Download the MP3 file from the URL
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; rv:93.0) Gecko/20100101 Firefox/93.0'}

    retries = 3
    delay = 5  # in seconds
    while retries > 0:
        response = requests.get(track, headers=headers)
        if response.status_code == 200:
            break
        retries -= 1
        time.sleep(delay)
    response = requests.get(track, headers=headers)
    print(f"{response}")
    # Save the downloaded content to a local file
    with open(local_file_path, 'wb') as f:
        f.write(response.content)
        return "sample.wav", track
   

def get_results(text_prompt,track_duration,gen_intensity,gen_mode):
    pat_token = get_pat_token()
    music = get_music(pat_token, text_prompt, track_duration, gen_intensity, gen_mode)
    return pat_token, music[0], music[1]

def get_prompts(uploaded_image, track_duration, gen_intensity, gen_mode, openai_api_key):
    print("calling clip interrogator")
    #prompt = img_to_text(uploaded_image, "ViT-L (best for Stable Diffusion 1.*)", "fast", fn_index=1)[0]
    
    prompt = img_to_text(uploaded_image, 'best', 4, fn_index=1)[0]
    print(prompt)
    clean_prompt = clean_text(prompt)
    print(f"prompt cleaned: {clean_prompt}")
    musical_prompt = 'You did not use any OpenAI API key to pimp your result :)'
    if openai_api_key is not None:
        gpt_adaptation = try_api(prompt, openai_api_key)
        if gpt_adaptation[0] != "oups":
            musical_prompt = gpt_adaptation[0]
            print(f"musical adapt: {musical_prompt}")
            music_result = get_results(musical_prompt, track_duration, gen_intensity, gen_mode)
        else:
            music_result = get_results(clean_prompt, track_duration, gen_intensity, gen_mode)
    else:
        music_result = get_results(clean_prompt, track_duration, gen_intensity, gen_mode)
    
    show_prompts = f"""
        CLIP Interrogator Caption: '{prompt}'
        β€”
        OpenAI Musical Adaptation: '{musical_prompt}'
        β€”
        Audio file link: {music_result[2]}
    """
    #wave_file = convert_mp3_to_wav(music_result[1])
    
    time.sleep(1)
    return gr.Textbox.update(value=show_prompts, visible=True), music_result[1], gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)

def try_api(message, openai_api_key):

    try:
        response = call_api(message, openai_api_key)
        return response, "<span class='openai_clear'>no error</span>"
    except openai.error.Timeout as e:
        #Handle timeout error, e.g. retry or log
        #print(f"OpenAI API request timed out: {e}")
        return "oups", f"<span class='openai_error'>OpenAI API request timed out: <br />{e}</span>"
    except openai.error.APIError as e:
        #Handle API error, e.g. retry or log
        #print(f"OpenAI API returned an API Error: {e}")
        return "oups", f"<span class='openai_error'>OpenAI API returned an API Error: <br />{e}</span>"
    except openai.error.APIConnectionError as e:
        #Handle connection error, e.g. check network or log
        #print(f"OpenAI API request failed to connect: {e}")
        return "oups", f"<span class='openai_error'>OpenAI API request failed to connect: <br />{e}</span>"
    except openai.error.InvalidRequestError as e:
        #Handle invalid request error, e.g. validate parameters or log
        #print(f"OpenAI API request was invalid: {e}")
        return "oups", f"<span class='openai_error'>OpenAI API request was invalid: <br />{e}</span>"
    except openai.error.AuthenticationError as e:
        #Handle authentication error, e.g. check credentials or log
        #print(f"OpenAI API request was not authorized: {e}")
        return "oups", f"<span class='openai_error'>OpenAI API request was not authorized: <br />{e}</span>"
    except openai.error.PermissionError as e:
        #Handle permission error, e.g. check scope or log
        #print(f"OpenAI API request was not permitted: {e}")
        return "oups", f"<span class='openai_error'>OpenAI API request was not permitted: <br />{e}</span>"
    except openai.error.RateLimitError as e:
        #Handle rate limit error, e.g. wait or log
        #print(f"OpenAI API request exceeded rate limit: {e}")
        return "oups", f"<span class='openai_error'>OpenAI API request exceeded rate limit: <br />{e}</span>"

def call_api(message, openai_api_key):

    instruction = "Convert in less than 200 characters this image caption to a very concise musical description with musical terms, as if you wanted to describe a musical ambiance, stricly in English"
          
    print("starting open ai")
    augmented_prompt = f"{instruction}: '{message}'."
    openai.api_key = openai_api_key
    
    response = openai.Completion.create(
        model="text-davinci-003",
        prompt=augmented_prompt,
        temperature=0.5,
        max_tokens=2048,
        top_p=1,
        frequency_penalty=0,
        presence_penalty=0.6
    )

    #print(response)

    #return str(response.choices[0].text).split("\n",2)[2]
    return str(response.choices[0].text).lstrip('\n')  


def get_track_by_tags(tags, pat, duration, gen_intensity, gen_mode, maxit=20):
    
    r = httpx.post('https://api-b2b.mubert.com/v2/RecordTrackTTM',
                   json={
                       "method": "RecordTrackTTM",
                       "params": {
                           "pat": pat,
                           "duration": duration,
                           "format": "wav",
                           "intensity":gen_intensity,
                           "tags": tags,
                           "mode": gen_mode
                       }
                   })

    rdata = json.loads(r.text)
    print(rdata)
    #assert rdata['status'] == 1, rdata['error']['text']
    trackurl = rdata['data']['tasks'][0]

    print('Generating track ', end='')
    for i in range(maxit):
        r = httpx.get(trackurl)
        if r.status_code == 200:
            return trackurl
        time.sleep(1)


def generate_track_by_prompt(pat, prompt, duration, gen_intensity, gen_mode):
    try:
        _, tags = get_tags_for_prompts(minilm, mubert_tags_embeddings, prompt)[0]
        result = get_track_by_tags(tags, pat, int(duration), gen_intensity, gen_mode)
        print(result)
        return result, ",".join(tags), "Success"
    except Exception as e:
        return None, "", str(e)

def convert_mp3_to_wav(mp3_filepath):
 
  wave_file="file.wav"
  
  sound = AudioSegment.from_mp3(mp3_filepath)
  sound.export(wave_file, format="wav")
  
  return wave_file

def remove_emoji(text):
    emoji_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', text)

def remove_nonalphanumeric(text):
    return re.sub(r'[^a-zA-Z0-9\s]', '', text)

def clean_text(text):
    clean_text = remove_nonalphanumeric(text)
    clean_text = remove_emoji(clean_text)
    clean_text = re.sub(r'\d+', '', clean_text) # Remove any number
    return clean_text

article = """
    
    <div class="footer">
        <p>
         
        Follow <a href="https://twitter.com/fffiloni" target="_blank">Sylvain Filoni</a> for future updates πŸ€—
        </p>
    </div>
    
    <div id="may-like-container" style="display: flex;justify-content: center;flex-direction: column;align-items: center;margin-bottom: 30px;">
        <p style="font-size: 0.8em;margin-bottom: 4px;">You may also like: </p>
        <div id="may-like" style="display: flex;flex-wrap: wrap;align-items: center;height: 20px;">
            <svg height="20" width="122" style="margin-left:4px;margin-bottom: 6px;">       
                 <a href="https://huggingface.co/spaces/fffiloni/spectrogram-to-music" target="_blank">
                    <image href="https://img.shields.io/badge/πŸ€— Spaces-Riffusion-blue" src="https://img.shields.io/badge/πŸ€— Spaces-Riffusion-blue.png" height="20"/>
                 </a>
            </svg>
        </div>
    </div>

    
"""

with gr.Blocks(css="style.css") as demo:
    with gr.Column(elem_id="col-container"):
        
        gr.HTML("""<div style="text-align: center; max-width: 700px; margin: 0 auto;">
                <div
                style="
                    display: inline-flex;
                    align-items: center;
                    gap: 0.8rem;
                    font-size: 1.75rem;
                "
                >
                <h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 5px;">
                    Image to Music
                </h1>
                </div>
                <p style="margin-bottom: 10px; font-size: 94%">
                Sends an image in to <a href="https://huggingface.co/spaces/pharma/CLIP-Interrogator" target="_blank">CLIP Interrogator</a>
                to generate a text prompt which is then run through 
                <a href="https://huggingface.co/Mubert" target="_blank">Mubert</a> text-to-music to generate music from the input image!
                </p>
            </div>""")
    
        input_img = gr.Image(type="filepath", elem_id="input-img")
        prompts_out = gr.Textbox(label="Text Captions", visible=False, elem_id="prompts_out", info="If player do not work, try to copy/paste the link in a new browser window")
        music_output = gr.Audio(label="Result", type="filepath", elem_id="music-output").style(height="5rem")
        #music_url = gr.Textbox(max_lines=1, info="If player do not work, try to copy/paste the link in a new browser window")
        #text_status = gr.Textbox(label="status")
        with gr.Group(elem_id="share-btn-container"):
            community_icon = gr.HTML(community_icon_html, visible=False)
            loading_icon = gr.HTML(loading_icon_html, visible=False)
            share_button = gr.Button("Share to community", elem_id="share-btn", visible=False)

        with gr.Accordion(label="Music Generation Options", open=False):
            openai_api_key = gr.Textbox(type="password", label="πŸ” Your OpenAI API Key (optional)", placeholder="sk-123abc...", info="You can use your OpenAI key to adapt CLIP Interrogator caption to a musical translation.")
            track_duration = gr.Slider(minimum=20, maximum=120, value=55, ustep=5, label="Track duration", elem_id="duration-inp")
            with gr.Row():
                gen_intensity = gr.Dropdown(choices=["low", "medium", "high"], value="medium", label="Intensity")
                gen_mode = gr.Radio(label="mode", choices=["track", "loop"], value="loop")
        
        generate = gr.Button("Generate Music from Image")

        gr.HTML(article)
    
    generate.click(get_prompts, inputs=[input_img,track_duration,gen_intensity,gen_mode, openai_api_key], outputs=[prompts_out, music_output, share_button, community_icon, loading_icon], api_name="i2m")
    share_button.click(None, [], [], _js=share_js)

demo.queue(max_size=32).launch()