import gradio as gr import os import time from moviepy.editor import * from share_btn import community_icon_html, loading_icon_html, share_js token = os.environ.get('HF_TOKEN') caption = gr.Blocks.load(name="spaces/laion/CoCa") audio_gen = gr.Blocks.load(name="spaces/haoheliu/audioldm-text-to-audio-generation") ph_message="If you're not happy with sound result, you can manually describe the scene depicted in your image :)" def input_changes(input_img): if input_img == None: return manual_cap.update(value="",placeholder=ph_message), caption_output.update(value=None), sound_output.update(value=None) else: cap = caption(input_img, "Beam search", 1.2, 0.5, 5, 20, fn_index=0) print("CoCa caption: '" + cap + "' • ") ph_update = "CoCa caption: '" + cap + "' • " return manual_cap.update(value="",placeholder=f"{ph_update}{ph_message}"), caption_output.update(value=cap), sound_output.update(value=None) def infer(image_input, manual_caption, duration_in, seed, caption_output): print(duration_in) if manual_caption == "": cap = caption_output #cap = caption(image_input, fn_index=0) #print("CoCa caption: '" + cap + "' • ") #ph_update = "CoCa caption: '" + cap + "' • " else: cap = manual_caption print("manual caption: " + cap) ph_update="" sound = audio_gen(cap, duration_in, 2.5, seed, 3, fn_index=0) print(sound) video = VideoFileClip(sound) audio = video.audio audio.write_audiofile("sound.mp3") #return cap, sound[1], gr.Textbox.update(placeholder=f"{ph_update}{ph_message}"), gr.Group.update(visible=True) return cap, "sound.mp3", gr.Group.update(visible=True) title = """

Image to Sound Effect

Convert an image to a corresponding sound effect generated through CoCa Image Captioning & AudioLDM

""" article = """

You may also like:

""" with gr.Blocks(css="style.css") as demo: with gr.Column(elem_id="col-container"): gr.HTML(title) input_img = gr.Image(type="filepath", elem_id="input-img") with gr.Column(): manual_cap = gr.Textbox(label="Manual Image description (optional)", lines=3, placeholder=ph_message) with gr.Row(): duration_in = gr.Slider(minimum=5, maximum=10, step=5, value=5, label="Duration") seed_in = gr.Slider(label="Seed", value=440, minimum=45, maximum=10000, step=1) caption_output = gr.Textbox(label="Caption", visible=False, elem_id="text-caption") sound_output = gr.Audio(label="Result", elem_id="sound-output") #debug = gr.Textbox() generate = gr.Button("Generate SFX from Image") with gr.Group(elem_id="share-btn-container", visible=False) as share_group: community_icon = gr.HTML(community_icon_html) loading_icon = gr.HTML(loading_icon_html) share_button = gr.Button("Share to community", elem_id="share-btn") gr.HTML(article) change_out = [manual_cap, caption_output, sound_output] input_img.change(input_changes, input_img, change_out, queue=False) generate.click(infer, inputs=[input_img, manual_cap, duration_in, seed_in, caption_output], outputs=[caption_output, sound_output, share_group], api_name="i2fx") share_button.click(None, [], [], _js=share_js) demo.queue(max_size=32).launch(debug=True)