src/pager.py

from bs4 import BeautifulSoup
from pydantic import BaseModel
import requests
from typing import Optional

class Pager(BaseModel):
    title: str
    text: str
    original_tags: list[str]

def is_valid_page(url):
    return True

def get_pager(url)->Optional[Pager]:
    try:
        req = requests.get(url)
        soup = BeautifulSoup(req.text, 'lxml')
        query = soup.find("div", class_="article-formatted-body")
        title = soup.title.string
        return Pager(title=title, text=query.get_text(), original_tags=["govno"])
    except:
        return None