Spaces:
Runtime error
Runtime error
| # Imports | |
| import gradio as gr | |
| import whisper | |
| from pytube import YouTube | |
| from transformers import pipeline, T5Tokenizer, T5ForConditionalGeneration | |
| from wordcloud import WordCloud | |
| class GradioInference: | |
| def __init__(self): | |
| # OpenAI's Whisper model sizes | |
| self.sizes = list(whisper._MODELS.keys()) | |
| # Whisper's available languages for ASR | |
| self.langs = ["none"] + sorted(list(whisper.tokenizer.LANGUAGES.values())) | |
| # Default size | |
| self.current_size = "base" | |
| # Default model size | |
| self.loaded_model = whisper.load_model(self.current_size) | |
| # Initialize Pytube Object | |
| self.yt = None | |
| # Initialize summary model | |
| self.summarizer = pipeline("summarization", model="facebook/bart-large-cnn") | |
| # Initialize VoiceLabT5 model and tokenizer | |
| self.keyword_model = T5ForConditionalGeneration.from_pretrained( | |
| "Voicelab/vlt5-base-keywords" | |
| ) | |
| self.keyword_tokenizer = T5Tokenizer.from_pretrained( | |
| "Voicelab/vlt5-base-keywords" | |
| ) | |
| # Sentiment Classifier | |
| self.classifier = pipeline("text-classification") | |
| def __call__(self, link, lang, size): | |
| """ | |
| Call the Gradio Inference python class. | |
| This class gets access to a YouTube video using python's library Pytube and downloads its audio. | |
| Then it uses the Whisper model to perform Automatic Speech Recognition (i.e Speech-to-Text). | |
| Once the function has the transcription of the video it proccess it to obtain: | |
| - Summary: using Facebook's BART transformer. | |
| - KeyWords: using VoiceLabT5 keyword extractor. | |
| - Sentiment Analysis: using Hugging Face's default sentiment classifier | |
| - WordCloud: using the wordcloud python library. | |
| """ | |
| if self.yt is None: | |
| self.yt = YouTube(link) | |
| # Pytube library to access to YouTube audio stream | |
| path = self.yt.streams.filter(only_audio=True)[0].download(filename="tmp.mp4") | |
| if lang == "none": | |
| lang = None | |
| if size != self.current_size: | |
| self.loaded_model = whisper.load_model(size) | |
| self.current_size = size | |
| # Transcribe the audio extracted from pytube | |
| results = self.loaded_model.transcribe(path, language=lang) | |
| # Perform summarization on the transcription | |
| transcription_summary = self.summarizer( | |
| results["text"], max_length=512, min_length=30, do_sample=False | |
| ) | |
| # Extract keywords using VoiceLabT5 | |
| task_prefix = "Keywords: " | |
| input_sequence = task_prefix + results["text"] | |
| input_ids = self.keyword_tokenizer( | |
| input_sequence, return_tensors="pt", truncation=False | |
| ).input_ids | |
| output = self.keyword_model.generate( | |
| input_ids, no_repeat_ngram_size=3, num_beams=4 | |
| ) | |
| predicted = self.keyword_tokenizer.decode(output[0], skip_special_tokens=True) | |
| keywords = [x.strip() for x in predicted.split(",") if x.strip()] | |
| # Sentiment label | |
| label = self.classifier(results["text"])[0]["label"] | |
| # Generate WordCloud object | |
| wordcloud = WordCloud().generate(results["text"]) | |
| # WordCloud image to display | |
| wordcloud_image = wordcloud.to_image() | |
| return ( | |
| results["text"], | |
| transcription_summary[0]["summary_text"], | |
| keywords, | |
| label, | |
| wordcloud_image, | |
| ) | |
| def populate_metadata(self, link): | |
| """ | |
| Access to the YouTube video title and thumbnail image to further display it | |
| params: | |
| - link: a YouTube URL. | |
| """ | |
| self.yt = YouTube(link) | |
| return self.yt.thumbnail_url, self.yt.title | |
| def from_audio_input(self, lang, size, audio_file): | |
| """ | |
| Call the Gradio Inference python class. | |
| Uses it directly the Whisper model to perform Automatic Speech Recognition (i.e Speech-to-Text). | |
| Once the function has the transcription of the video it proccess it to obtain: | |
| - Summary: using Facebook's BART transformer. | |
| - KeyWords: using VoiceLabT5 keyword extractor. | |
| - Sentiment Analysis: using Hugging Face's default sentiment classifier | |
| - WordCloud: using the wordcloud python library. | |
| """ | |
| if lang == "none": | |
| lang = None | |
| if size != self.current_size: | |
| self.loaded_model = whisper.load_model(size) | |
| self.current_size = size | |
| results = self.loaded_model.transcribe(audio_file, language=lang) | |
| # Perform summarization on the transcription | |
| transcription_summary = self.summarizer( | |
| results["text"], max_length=512, min_length=30, do_sample=False | |
| ) | |
| # Extract keywords using VoiceLabT5 | |
| task_prefix = "Keywords: " | |
| input_sequence = task_prefix + results["text"] | |
| input_ids = self.keyword_tokenizer( | |
| input_sequence, return_tensors="pt", truncation=False | |
| ).input_ids | |
| output = self.keyword_model.generate( | |
| input_ids, no_repeat_ngram_size=3, num_beams=4 | |
| ) | |
| predicted = self.keyword_tokenizer.decode(output[0], skip_special_tokens=True) | |
| keywords = [x.strip() for x in predicted.split(",") if x.strip()] | |
| # Sentiment label | |
| label = self.classifier(results["text"])[0]["label"] | |
| # WordCloud object | |
| wordcloud = WordCloud().generate( | |
| results["text"] | |
| ) | |
| wordcloud_image = wordcloud.to_image() | |
| return ( | |
| results["text"], | |
| transcription_summary[0]["summary_text"], | |
| keywords, | |
| label, | |
| wordcloud_image, | |
| ) | |
| gio = GradioInference() | |
| title = "Youtube Insights" | |
| description = "Your AI-powered video analytics tool" | |
| block = gr.Blocks() | |
| with block as demo: | |
| gr.HTML( | |
| """ | |
| <div style="text-align: center; max-width: 500px; margin: 0 auto;"> | |
| <div> | |
| <h1>Youtube <span style="color: red;">Insights</span> 📹</h1> | |
| </div> | |
| <p style="margin-bottom: 10px; font-size: 94%"> | |
| Your AI-powered video analytics tool | |
| </p> | |
| </div> | |
| """ | |
| ) | |
| with gr.Group(): | |
| with gr.Tab("From YouTube"): | |
| with gr.Box(): | |
| with gr.Row().style(equal_height=True): | |
| size = gr.Dropdown( | |
| label="Model Size", choices=gio.sizes, value="base" | |
| ) | |
| lang = gr.Dropdown( | |
| label="Language (Optional)", choices=gio.langs, value="none" | |
| ) | |
| link = gr.Textbox( | |
| label="YouTube Link", placeholder="Enter YouTube link..." | |
| ) | |
| title = gr.Label(label="Video Title") | |
| with gr.Row().style(equal_height=True): | |
| img = gr.Image(label="Thumbnail") | |
| text = gr.Textbox( | |
| label="Transcription", | |
| placeholder="Transcription Output...", | |
| lines=10, | |
| ).style(show_copy_button=True, container=True) | |
| with gr.Row().style(equal_height=True): | |
| summary = gr.Textbox( | |
| label="Summary", placeholder="Summary Output...", lines=5 | |
| ).style(show_copy_button=True, container=True) | |
| keywords = gr.Textbox( | |
| label="Keywords", placeholder="Keywords Output...", lines=5 | |
| ).style(show_copy_button=True, container=True) | |
| label = gr.Label(label="Sentiment Analysis") | |
| wordcloud_image = gr.Image() | |
| with gr.Row().style(equal_height=True): | |
| clear = gr.ClearButton( | |
| [link, title, img, text, summary, keywords, label, wordcloud_image], scale=1 | |
| ) | |
| btn = gr.Button("Get video insights", variant="primary", scale=1) | |
| btn.click( | |
| gio, | |
| inputs=[link, lang, size], | |
| outputs=[text, summary, keywords, label, wordcloud_image], | |
| ) | |
| if link: | |
| link.change(gio.populate_metadata, inputs=[link], outputs=[img, title]) | |
| with gr.Tab("From Audio file"): | |
| with gr.Box(): | |
| with gr.Row().style(equal_height=True): | |
| size = gr.Dropdown( | |
| label="Model Size", choices=gio.sizes, value="base" | |
| ) | |
| lang = gr.Dropdown( | |
| label="Language (Optional)", choices=gio.langs, value="none" | |
| ) | |
| audio_file = gr.Audio(type="filepath") | |
| with gr.Row().style(equal_height=True): | |
| text = gr.Textbox( | |
| label="Transcription", | |
| placeholder="Transcription Output...", | |
| lines=10, | |
| ).style(show_copy_button=True, container=False) | |
| with gr.Row().style(equal_height=True): | |
| summary = gr.Textbox( | |
| label="Summary", placeholder="Summary Output", lines=5 | |
| ) | |
| keywords = gr.Textbox( | |
| label="Keywords", placeholder="Keywords Output", lines=5 | |
| ) | |
| label = gr.Label(label="Sentiment Analysis") | |
| wordcloud_image = gr.Image() | |
| with gr.Row().style(equal_height=True): | |
| clear = gr.ClearButton([audio_file,text, summary, keywords, label, wordcloud_image], scale=1) | |
| btn = gr.Button( | |
| "Get video insights", variant="primary", scale=1 | |
| ) | |
| btn.click( | |
| gio.from_audio_input, | |
| inputs=[lang, size, audio_file], | |
| outputs=[text, summary, keywords, label, wordcloud_image], | |
| ) | |
| with block: | |
| gr.Markdown("### Video Examples") | |
| gr.Examples(["https://www.youtube.com/shorts/xDNzz8yAH7I"], inputs=link) | |
| gr.Markdown("About the app:") | |
| with gr.Accordion("What is YouTube Insights?", open=False): | |
| gr.Markdown( | |
| "YouTube Insights is a tool developed with academic purposes only, that creates summaries, keywords and sentiments analysis based on YouTube videos or user audio files." | |
| ) | |
| with gr.Accordion("How does it work?", open=False): | |
| gr.Markdown( | |
| "Works by using OpenAI's Whisper, BART for summarization and VoiceLabT5 for Keyword Extraction." | |
| ) | |
| gr.HTML( | |
| """ | |
| <div style="text-align: center; max-width: 500px; margin: 0 auto;"> | |
| <p style="margin-bottom: 10px; font-size: 96%"> | |
| 2023 Master in Big Data & Data Science - Universidad Complutense de Madrid | |
| </p> | |
| </div> | |
| """ | |
| ) | |
| demo.launch() | |