nlg models removal

app.py

@@ -1,14 +1,13 @@
 import gradio as gr
 
-from models import DST_MODELS, NLG_MODELS, PIPELINES
+from models import DST_MODELS, PIPELINES
 
 
 def predict(text: str, model_name: str) -> str:
     return PIPELINES[model_name](text)
 
 
-with gr.Blocks(title="CLARIN-PL Dialogue System Modules") as demo:
-    gr.Markdown("Dialogue State Tracking Modules")
+with gr.Blocks(title="CLARIN-PL DST Modules") as demo:
     for model_name in DST_MODELS:
         with gr.Row():
             gr.Markdown(f"## {model_name}")
@@ -21,18 +20,5 @@ with gr.Blocks(title="CLARIN-PL Dialogue System Modules") as demo:
             predict_button.click(fn=predict, inputs=[text_input, model_name_component], outputs=output)
 
 
-    gr.Markdown("Natural Language Generation / Paraphrasing Modules")
-    for model_name in NLG_MODELS:
-        with gr.Row():
-            gr.Markdown(f"## {model_name}")
-            model_name_component = gr.Textbox(value=model_name, visible=False)
-        with gr.Row():
-            text_input = gr.Textbox(label="Input Text", value=NLG_MODELS[model_name]["default_input"])
-            output = gr.Textbox(label="Slot Value", value="")
-        with gr.Row():
-            predict_button = gr.Button("Predict")
-            predict_button.click(fn=predict, inputs=[text_input, model_name_component], outputs=output)
-
-
 demo.queue(concurrency_count=3)
 demo.launch()

models.py

@@ -2,7 +2,7 @@ import os
 from typing import Any, Dict
 
 from transformers import (Pipeline, T5ForConditionalGeneration, T5Tokenizer,
-                          pipeline, AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer)
+                          pipeline)
 
 auth_token = os.environ.get("CLARIN_KNEXT")
 
@@ -60,48 +60,6 @@ DST_MODELS: Dict[str, Dict[str, Any]] = {
 }
 
 
-DEFAULT_ENCODER_DECODER_INPUT_EN = "The alarm is set for 6 am. The alarm's name is name \"Get up\"."
-DEFAULT_DECODER_ONLY_INPUT_EN = f"[BOS]{DEFAULT_ENCODER_DECODER_INPUT_EN}[SEP]"
-DEFAULT_ENCODER_DECODER_INPUT_PL = "Alarm jest o godzinie 6 rano. Alarm ma nazwę \"Obudź się\"."
-DEFAULT_DECODER_ONLY_INPUT_PL = f"[BOS]{DEFAULT_ENCODER_DECODER_INPUT_PL}[SEP]"
-
-
-
-NLG_MODELS: Dict[str, Dict[str, Any]] = {
-    # English
-    "t5-large": {
-        "model": AutoModelForSeq2SeqLM.from_pretrained("clarin-knext/utterance-rewriting-t5-large", use_auth_token=auth_token),
-        "tokenizer": AutoTokenizer.from_pretrained("clarin-knext/utterance-rewriting-t5-large", use_auth_token=auth_token),
-        "default_input": DEFAULT_ENCODER_DECODER_INPUT_EN,
-    },
-    "en-mt5-large": {
-        "model": AutoModelForSeq2SeqLM.from_pretrained("clarin-knext/utterance-rewriting-en-mt5-large", use_auth_token=auth_token),
-        "tokenizer": AutoTokenizer.from_pretrained("clarin-knext/utterance-rewriting-en-mt5-large", use_auth_token=auth_token),
-        "default_input": DEFAULT_ENCODER_DECODER_INPUT_EN,
-    },
-    "gpt2": {
-        "model": AutoModelForCausalLM.from_pretrained("clarin-knext/utterance-rewriting-gpt2", use_auth_token=auth_token),
-        "tokenizer": AutoTokenizer.from_pretrained("clarin-knext/utterance-rewriting-gpt2", use_auth_token=auth_token),
-        "default_input": DEFAULT_DECODER_ONLY_INPUT_EN,
-    },
-
-    "pt5-large": {
-        "model": AutoModelForSeq2SeqLM.from_pretrained("clarin-knext/utterance-rewriting-pt5-large", use_auth_token=auth_token),
-        "tokenizer": AutoTokenizer.from_pretrained("clarin-knext/utterance-rewriting-pt5-large", use_auth_token=auth_token),
-        "default_input": DEFAULT_ENCODER_DECODER_INPUT_PL,
-    },
-    "pl-mt5-large": {
-        "model": AutoModelForSeq2SeqLM.from_pretrained("clarin-knext/utterance-rewriting-pl-mt5-large", use_auth_token=auth_token),
-        "tokenizer": AutoTokenizer.from_pretrained("clarin-knext/utterance-rewriting-pl-mt5-large", use_auth_token=auth_token),
-        "default_input": DEFAULT_ENCODER_DECODER_INPUT_PL,
-    },
-    "polish-gpt2": {
-        "model": AutoModelForCausalLM.from_pretrained("clarin-knext/utterance-rewriting-polish-gpt2", use_auth_token=auth_token),
-        "tokenizer": AutoTokenizer.from_pretrained("clarin-knext/utterance-rewriting-polish-gpt2", use_auth_token=auth_token),
-        "default_input": DEFAULT_DECODER_ONLY_INPUT_PL,
-    },
-}
-
 PIPELINES: Dict[str, Pipeline] = {
     model_name: pipeline(
         "text2text-generation", model=DST_MODELS[model_name]["model"], tokenizer=DST_MODELS[model_name]["tokenizer"]