import gradio as gr import os import time from moviepy.editor import * from share_btn import community_icon_html, loading_icon_html, share_js token = os.environ.get('HF_TOKEN') caption = gr.Blocks.load(name="spaces/laion/CoCa") audio_gen = gr.Blocks.load(name="spaces/haoheliu/audioldm-text-to-audio-generation") ph_message="If you're not happy with sound result, you can manually describe the scene depicted in your image :)" def input_changes(input_img): if input_img == None: return manual_cap.update(value="",placeholder=ph_message), caption_output.update(value=None), sound_output.update(value=None) else: cap = caption(input_img, "Beam search", 1.2, 0.5, 5, 20, fn_index=0) print("CoCa caption: '" + cap + "' • ") ph_update = "CoCa caption: '" + cap + "' • " return manual_cap.update(value="",placeholder=f"{ph_update}{ph_message}"), caption_output.update(value=cap), sound_output.update(value=None) def infer(image_input, manual_caption, duration_in, seed, caption_output): print(duration_in) if manual_caption == "": cap = caption_output #cap = caption(image_input, fn_index=0) #print("CoCa caption: '" + cap + "' • ") #ph_update = "CoCa caption: '" + cap + "' • " else: cap = manual_caption print("manual caption: " + cap) ph_update="" sound = audio_gen(cap, duration_in, 2.5, seed, 3, fn_index=0) print(sound) video = VideoFileClip(sound) audio = video.audio audio.write_audiofile("sound.mp3") #return cap, sound[1], gr.Textbox.update(placeholder=f"{ph_update}{ph_message}"), gr.Group.update(visible=True) return cap, "sound.mp3", gr.Group.update(visible=True) title = """
Convert an image to a corresponding sound effect generated through CoCa Image Captioning & AudioLDM
You may also like: