m

.streamlit/config.toml

@@ -0,0 +1,4 @@
+[theme]
+base="dark"
+font="serif"
+primaryColor="purple"

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.12
+WORKDIR .
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+EXPOSE 5000
+
+
+CMD ["python", "-m", "streamlit", "app.py"]

app.py

@@ -0,0 +1,42 @@
+import streamlit as st
+from src.pager import get_pager
+from src.summarizer import Summarizer
+import torch
+
+
+torch.classes.__path__ = []
+
+@st.cache_resource
+def GetSummarizer():
+    return Summarizer()
+
+
+url = st.text_input("Please enter your habr article url...")
+text = st.text_input("...or paste text here:strawberry:")
+
+def handle_sum_text(sum_text):
+    return ['#' + x for x in sum_text.split()]
+
+def url_callback():
+    summarizer = GetSummarizer()
+    pager = get_pager(url)
+    if pager is not None:
+        st.title(pager.title)
+        sum_text = summarizer.summarize(pager.text[:1000])
+        st.write("Okay, there your tags :sunglasses:")
+        for chunk in handle_sum_text(sum_text):
+            st.badge(chunk, icon=":material/check:", color="green")
+    st.title(":shit: Слушай ну нормально же общались")
+
+def generator_callback():
+    summarizer = GetSummarizer()
+    st.title("Your AWESOME:heart: article")
+    sum_text = summarizer.summarize(text[:1000])
+    st.write("Okay, there your #tags :sunglasses:")
+    for chunk in handle_sum_text(sum_text):
+        st.badge(chunk, icon=":material/check:", color="green")
+
+
+st.button("Describe Habr Article", on_click=url_callback)
+
+st.button("Describe text", on_click=generator_callback)

main.ipynb

@@ -0,0 +1,83 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/shaenazar/anaconda3/envs/dsenv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
+    "\n",
+    "model_name = \"sarahai/ruT5-base-summarizer\"\n",
+    "model_path = \"data/checkpoint\"\n",
+    "\n",
+    "model = T5ForConditionalGeneration.from_pretrained(model_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "model.safetensors: 100%|██████████| 892M/892M [01:13<00:00, 12.2MB/s]  \n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/ShaeNaZar/YsdaSummarizer/commit/fbd9cbe753c47653b6418165c948f69dc160954e', commit_message='Upload T5ForConditionalGeneration', commit_description='', oid='fbd9cbe753c47653b6418165c948f69dc160954e', pr_url=None, repo_url=RepoUrl('https://huggingface.co/ShaeNaZar/YsdaSummarizer', endpoint='https://huggingface.co', repo_type='model', repo_id='ShaeNaZar/YsdaSummarizer'), pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.push_to_hub(\"ShaeNaZar/YsdaSummarizer\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "dsenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -0,0 +1,4 @@
+bs4
+streamlit
+torch
+transformers

src/pager.py

@@ -0,0 +1,22 @@
+from bs4 import BeautifulSoup
+from pydantic import BaseModel
+import requests
+from typing import Optional
+
+class Pager(BaseModel):
+    title: str
+    text: str
+    original_tags: list[str]
+
+def is_valid_page(url):
+    return True
+
+def get_pager(url)->Optional[Pager]:
+    try:
+        req = requests.get(url)
+        soup = BeautifulSoup(req.text, 'lxml')
+        query = soup.find("div", class_="article-formatted-body")
+        title = soup.title.string
+        return Pager(title=title, text=query.get_text(), original_tags=["govno"])
+    except:
+        return None

src/summarizer.py

@@ -0,0 +1,16 @@
+import torch
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+
+class Summarizer:
+    def __init__(self, device="cpu"):
+        model_name = "sarahai/ruT5-base-summarizer"
+        model_path = "ShaeNaZar/YsdaSummarizer"
+        self.device = device
+        self.tokenizer  = T5Tokenizer.from_pretrained(model_name)
+        self.model = T5ForConditionalGeneration.from_pretrained(model_path)
+
+    def summarize(self, text):
+        input_ids = self.tokenizer(text, return_tensors="pt").input_ids.to(self.device)
+        outputs = self.model.generate(input_ids, max_length=20, min_length=20, length_penalty=2.0, num_beams=5, early_stopping=True)
+
+        return self.tokenizer.decode(outputs[0], skip_special_tokens=True)