File: setup.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Note: To use the 'upload' functionality of this file, you must:
#   $ pipenv install twine --dev

import io
import os
import sys
from shutil import rmtree

from setuptools import Command, setup

# Package meta-data.
NAME = "yyets"
DESCRIPTION = "https://yyets.click/ wrapper"
URL = "https://github.com/tgbot-collection/YYeTsBot"
EMAIL = "benny.think@gmail.com"
AUTHOR = "BennyThink"
REQUIRES_PYTHON = ">=3.6.0"
VERSION = "1.0.1"

# What packages are required for this module to be executed?
REQUIRED = ["requests"]

# What packages are optional?
EXTRAS = {
    # 'fancy feature': ['django'],
}

# The rest you shouldn't have to touch too much :)
# ------------------------------------------------
# Except, perhaps the License and Trove Classifiers!
# If you do change the License, remember to change the Trove Classifier for that!

here = os.path.abspath(os.path.dirname(__file__))

# Import the README and use it as the long-description.
# Note: this will only work if 'README.md' is present in your MANIFEST.in file!
try:
    with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f:
        long_description = "\n" + f.read()
except FileNotFoundError:
    long_description = DESCRIPTION

# Load the package's __version__.py module as a dictionary.
about = {}
if not VERSION:
    project_slug = NAME.lower().replace("-", "_").replace(" ", "_")
    with open(os.path.join(here, project_slug, "__version__.py")) as f:
        exec(f.read(), about)
else:
    about["__version__"] = VERSION


class UploadCommand(Command):
    """Support setup.py upload."""

    description = "Build and publish the package."
    user_options = []

    @staticmethod
    def status(s):
        """Prints things in bold."""
        print("\033[1m{0}\033[0m".format(s))

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        try:
            self.status("Removing previous builds…")
            rmtree(os.path.join(here, "dist"))
        except OSError:
            pass

        self.status("Building Source and Wheel (universal) distribution…")
        os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable))

        self.status("Uploading the package to PyPI via Twine…")
        os.system("twine upload dist/*")

        self.status("Pushing git tags…")
        os.system("git tag v{0}".format(about["__version__"]))
        os.system("git push --tags")

        sys.exit()


# Where the magic happens:
setup(
    name=NAME,
    version=about["__version__"],
    description=DESCRIPTION,
    long_description=long_description,
    long_description_content_type="text/markdown",
    author=AUTHOR,
    author_email=EMAIL,
    python_requires=REQUIRES_PYTHON,
    url=URL,
    # packages=find_packages(exclude=["tests", "*.tests", "*.tests.*", "tests.*"]),
    # If your package is a single module, use this instead of 'packages':
    packages=["yyets"],
    # entry_points={
    #     'console_scripts': ['mycli=mymodule:cli'],
    # },
    install_requires=REQUIRED,
    extras_require=EXTRAS,
    include_package_data=True,
    license="MIT",
    classifiers=[
        # Trove classifiers
        # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: Implementation :: CPython",
        "Programming Language :: Python :: Implementation :: PyPy",
    ],
    # $ setup.py publish support.
    cmdclass={
        "upload": UploadCommand,
    },
)



File: yyetsbot/fansub.py

# coding: utf-8
# YYeTsBot - fansub.py
# 2019/8/15 18:30

__author__ = "Benny <benny.think@gmail.com>"

import contextlib
import hashlib
import json
import logging
import os
import pickle
import re
import sys

import fakeredis
import pymongo
import redis
import requests
from bs4 import BeautifulSoup
from bson.objectid import ObjectId

from config import (
    BD2020_SEARCH,
    FANSUB_ORDER,
    FIX_SEARCH,
    MONGO,
    NEWZMZ_RESOURCE,
    NEWZMZ_SEARCH,
    REDIS,
    WORKERS,
    DISCUSS,
    XL720_SEARCH,
    ZHUIXINFAN_RESOURCE,
    ZHUIXINFAN_SEARCH,
)

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(filename)s [%(levelname)s]: %(message)s")

session = requests.Session()
ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36"
session.headers.update({"User-Agent": ua})

this_module = sys.modules[__name__]


class Redis:
    def __init__(self):
        if os.getenv("DISABLE_REDIS"):
            self.r = fakeredis.FakeStrictRedis()
        else:
            self.r = redis.StrictRedis(host=REDIS, decode_responses=True)

    def __del__(self):
        self.r.close()

    @classmethod
    def preview_cache(cls, timeout=3600 * 24):
        def func(fun):
            def inner(*args, **kwargs):
                search_text = args[1]
                cache_value = cls().r.get(search_text)
                if cache_value:
                    logging.info("🎉 Preview cache hit for %s %s", fun, search_text)
                    return json.loads(cache_value)
                else:
                    logging.info("😱 Preview cache expired. Running %s %s", fun, search_text)
                    res = fun(*args, **kwargs)
                    # if res len is 1, then it means we have no search result at all.
                    if len(res) != 1:
                        json_str = json.dumps(res, ensure_ascii=False)
                        cls().r.set(search_text, json_str, ex=timeout)
                    # save hash->url mapping
                    res_copy = res.copy()
                    res_copy.pop("class")
                    for url_hash, value in res_copy.items():
                        cls().r.hset(url_hash, mapping=value)

                    return res

            return inner

        return func

    @classmethod
    def result_cache(cls, timeout=3600 * 24):
        def func(fun):
            def inner(*args, **kwargs):
                # this method will convert hash to url
                url_or_hash = args[1]
                if re.findall(r"http[s]?://", url_or_hash):
                    # means this is a url
                    url_hash = hashlib.sha1(url_or_hash.encode("u8")).hexdigest()
                    cls().r.hset(url_hash, mapping={"url": url_or_hash})
                else:
                    # this is cache, retrieve real url from redis
                    url_or_hash = cls().r.hget(url_or_hash, "url")
                    if not url_or_hash:
                        url_or_hash = ""

                url = url_or_hash
                del url_or_hash
                cache_value = cls().r.hgetall(url)
                if cache_value:
                    logging.info("🎉 Result cache hit for %s %s", fun, url)
                    return cache_value
                else:
                    logging.info("😱 Result cache expired. Running %s %s", fun, url)
                    new_args = (args[0], url)
                    res = fun(*new_args, **kwargs)
                    # we must have an result for it,
                    cls().r.hset(url, mapping=res)
                    cls().r.expire(url, timeout)

                    return res

            return inner

        return func


class BaseFansub:
    """
    all the subclass should implement three kinds of methods:
    1. online search, contains preview for bot and complete result
    2. login and check (set pass if not applicable)
    3. search_result as this is critical for bot to draw markup

    """

    cookie_file = None

    def __init__(self):
        self.redis = Redis().r

    @property
    def id(self):
        # implement how to get the unique id for this resource
        return None

    def get_html(self, link: str, encoding=None) -> str:
        # return html text of search page
        with session.get(link) as r:
            if encoding is not None:
                r.encoding = encoding
            html = r.text
        logging.info("[%s] [%s] Searching  for %s", self.__class__.__name__, r.status_code, link)
        return html

    def search_preview(self, search_text: str) -> dict:
        # try to retrieve critical information from html
        # this result must return to bot for manual selection
        # {"url1": "name1", "url2": "name2", "source":"yyets"}
        pass

    def search_result(self, url_or_hash: str) -> dict:
        """
        This will happen when user click one of the button, only by then we can know the resource link
        From the information above, try to get a detail dict structure.
        This method should check cache first if applicable
        This method should set self.link and self.data
        :param url_or_hash: url or hash.
        :return:    {"all": dict_result, "share": share_link, "cnname": cnname}

        """
        pass

    def __login_check(self):
        pass

    def __manual_login(self):
        pass

    def __save_cookies__(self, requests_cookiejar):
        with open(self.cookie_file, "wb") as f:
            pickle.dump(requests_cookiejar, f)

    def __load_cookies__(self):
        with open(self.cookie_file, "rb") as f:
            return pickle.load(f)


class YYeTsOffline(BaseFansub):
    def __init__(self, db="zimuzu", col="yyets"):
        super().__init__()
        self.mongo = pymongo.MongoClient(host=MONGO)
        self.db = self.mongo[db]
        self.collection = self.db[col]

    @Redis.preview_cache(60)
    def search_preview(self, search_text: str) -> dict:
        logging.info("[%s] Loading offline data from MongoDB...", self.__class__.__name__)

        projection = {"_id": False, "data.info": True}
        data = self.collection.find(
            {
                "$or": [
                    {"data.info.cnname": {"$regex": f".*{search_text}.*", "$options": "i"}},
                    {"data.info.enname": {"$regex": f".*{search_text}.*", "$options": "i"}},
                    {"data.info.aliasname": {"$regex": f".*{search_text}.*", "$options": "i"}},
                ]
            },
            projection,
        )
        results = {}
        for item in data:
            info = item["data"]["info"]
            url = WORKERS.format(info["id"])
            url_hash = hashlib.sha1(url.encode("u8")).hexdigest()
            all_name = info["cnname"] + info["enname"] + info["aliasname"]
            results[url_hash] = {"name": all_name, "url": url, "class": self.__class__.__name__}

        logging.info("[%s] Offline resource search complete", self.__class__.__name__)

        comments = self.db["comment"].find({"content": {"$regex": f".*{search_text}.*", "$options": "i"}})
        for c in comments:
            if c["resource_id"] == 233:
                url = DISCUSS.format(str(c["_id"]))
            else:
                url = WORKERS + "#{}".format(c["resource_id"], str(c["_id"]))
            url_hash = hashlib.sha1(url.encode("u8")).hexdigest()
            all_name = c["content"]
            results[url_hash] = {"name": all_name, "url": url, "class": self.__class__.__name__}
        logging.info("[%s] Offline comment search complete", self.__class__.__name__)

        results["class"] = self.__class__.__name__
        return results

    @Redis.result_cache(10 * 60)
    def search_result(self, resource_url) -> dict:
        # yyets offline

        # resource: https://yyets.click/resource.html?id=37089
        # comment: 'https://yyets.click/resource.html?id=233#61893ae51e9152e43fa24124'
        if "#" in resource_url:
            cid = resource_url.split("#")[1]
            data: dict = self.db["comment"].find_one(
                {"_id": ObjectId(cid)}, {"_id": False, "ip": False, "type": False, "children": False, "browser": False}
            )
            share = resource_url
            name = f"{data['username']} 的分享"
            t = "comment"
        else:
            rid = resource_url.split("id=")[1]
            data: dict = self.collection.find_one({"data.info.id": int(rid)}, {"_id": False})
            name = data["data"]["info"]["cnname"]
            share = WORKERS.format(rid)
            t = "resource"

        return {"all": json.dumps(data, ensure_ascii=False, indent=4), "share": share, "cnname": name, "type": t}

    def __del__(self):
        self.mongo.close()


class ZimuxiaOnline(BaseFansub):
    @Redis.preview_cache()
    def search_preview(self, search_text: str) -> dict:
        # zimuxia online
        search_url = FIX_SEARCH.format(kw=search_text)
        html_text = self.get_html(search_url)
        logging.info("[%s] Parsing html...", self.__class__.__name__)
        soup = BeautifulSoup(html_text, "html.parser")
        link_list = soup.find_all("h2", class_="post-title")

        dict_result = {}
        for link in link_list:
            # TODO wordpress search content and title, some cases it would be troublesome
            url = link.a["href"]
            url_hash = hashlib.sha1(url.encode("u8")).hexdigest()
            name = link.a.text
            dict_result[url_hash] = {"url": url, "name": name, "class": self.__class__.__name__}
        dict_result["class"] = self.__class__.__name__
        return dict_result

    @Redis.result_cache()
    def search_result(self, resource_url: str) -> dict:
        # zimuxia online
        logging.info("[%s] Loading detail page %s", self.__class__.__name__, resource_url)
        html = self.get_html(resource_url)
        soup = BeautifulSoup(html, "html.parser")
        cnname = soup.title.text.split("|")[0]
        return {"all": html, "share": resource_url, "cnname": cnname, "type": "resource"}


class ZhuixinfanOnline(BaseFansub):
    @Redis.preview_cache()
    def search_preview(self, search_text: str) -> dict:
        # zhuixinfan online
        search_link = ZHUIXINFAN_SEARCH.format(search_text)
        html_text = self.get_html(search_link)
        logging.info("[%s] Parsing html...", self.__class__.__name__)
        soup = BeautifulSoup(html_text, "html.parser")
        link_list = soup.find_all("ul", class_="resource_list")

        dict_result = {}
        for li in link_list:
            for link in li:
                with contextlib.suppress(AttributeError):
                    name = link.dd.text
                    url = ZHUIXINFAN_RESOURCE.format(link.dd.a["href"])
                    url_hash = hashlib.sha1(url.encode("u8")).hexdigest()
                    dict_result[url_hash] = {"url": url, "name": name, "class": self.__class__.__name__}
        dict_result["class"] = self.__class__.__name__
        return dict_result

    @Redis.result_cache()
    def search_result(self, url: str) -> dict:
        # zhuixinfan online
        # don't worry, url_hash will become real url
        logging.info("[%s] Loading detail page %s", self.__class__.__name__, url)
        html = self.get_html(url, "utf-8")
        # 解析获得cnname等信息
        soup = BeautifulSoup(html, "html.parser")
        cnname = soup.title.text.split("_")[0]
        return {"all": html, "share": url, "cnname": cnname, "type": "resource"}


class NewzmzOnline(BaseFansub):
    @Redis.preview_cache()
    def search_preview(self, search_text: str) -> dict:
        # zhuixinfan online
        search_link = NEWZMZ_SEARCH.format(search_text)
        html_text = self.get_html(search_link)
        search_response = json.loads(html_text)

        dict_result = {}
        for item in search_response["data"]:
            url = NEWZMZ_RESOURCE.format(item["link_url"].split("-")[1])
            url_hash = hashlib.sha1(url.encode("u8")).hexdigest()
            dict_result[url_hash] = {
                "url": url,
                "name": item["name"] + item["name_eng"],
                "class": self.__class__.__name__,
            }
        dict_result["class"] = self.__class__.__name__
        return dict_result

    @Redis.result_cache()
    def search_result(self, url: str) -> dict:
        logging.info("[%s] Loading detail page %s", self.__class__.__name__, url)
        html = self.get_html(url)
        # 解析获得cnname等信息
        soup = BeautifulSoup(html, "html.parser")
        cnname = soup.title.text.split("-")[0]
        return {"all": html, "share": url, "cnname": cnname, "type": "resource"}


class BD2020(NewzmzOnline):
    @Redis.preview_cache()
    def search_preview(self, search_text: str) -> dict:
        search_link = BD2020_SEARCH.format(search_text)
        html_text = self.get_html(search_link)
        logging.info("[%s] Parsing html...", self.__class__.__name__)
        soup = BeautifulSoup(html_text, "html.parser")
        link_list = soup.find_all("li", class_="list-item")

        dict_result = {}
        for item in link_list:
            name = item.div.a.text.strip()
            url = item.div.a["href"]
            url_hash = hashlib.sha1(url.encode("u8")).hexdigest()
            dict_result[url_hash] = {"url": url, "name": name, "class": self.__class__.__name__}

        dict_result["class"] = self.__class__.__name__
        return dict_result


class XL720(BD2020):
    @Redis.preview_cache()
    def search_preview(self, search_text: str) -> dict:
        search_link = XL720_SEARCH.format(search_text)
        html_text = self.get_html(search_link)
        logging.info("[%s] Parsing html...", self.__class__.__name__)
        soup = BeautifulSoup(html_text, "html.parser")

        dict_result = {}
        link_list = soup.find_all("div", class_="post clearfix")
        for item in link_list:
            name = re.sub(r"\s", "", item.h3.a.text.strip())
            url = item.h3.a["href"]
            url_hash = hashlib.sha1(url.encode("u8")).hexdigest()
            dict_result[url_hash] = {"url": url, "name": name, "class": self.__class__.__name__}

        dict_result["class"] = self.__class__.__name__
        return dict_result

    @Redis.result_cache()
    def search_result(self, url: str) -> dict:
        logging.info("[%s] Loading detail page %s", self.__class__.__name__, url)
        html = self.get_html(url)
        soup = BeautifulSoup(html, "html.parser")
        cnname = soup.title.text.split("迅雷下载")[0]
        return {"all": html, "share": url, "cnname": cnname, "type": "resource"}


class FansubEntrance(BaseFansub):
    order = FANSUB_ORDER.split(",")

    def search_preview(self, search_text: str) -> dict:
        class_ = "聪明机智温柔可爱善良的Benny"
        for sub_str in self.order:
            logging.info("Looping from %s", sub_str)
            fc = globals().get(sub_str)
            result = fc().search_preview(search_text)
            # this result contains source:sub, so we'll pop and add it
            class_ = result.pop("class")
            if result:
                logging.info("Result hit in %s %s", sub_str, fc)
                FansubEntrance.fansub_class = fc
                result["class"] = class_
                return result

        return {"class": class_}

    def search_result(self, resource_url_hash: str) -> dict:
        # entrance
        cache_data = self.redis.hgetall(resource_url_hash)
        resource_url = cache_data["url"]
        class_name = cache_data["class"]
        fc = globals().get(class_name)
        return fc().search_result(resource_url)


# we'll check if FANSUB_ORDER is correct. Must put it here, not before.
for fs in FANSUB_ORDER.split(","):
    if globals().get(fs) is None:
        raise NameError(f"FANSUB_ORDER is incorrect! {fs}")


# Commands can use latin letters, numbers and underscores. yyets_offline
def class_to_tg(sub_class: str):
    trans = {"Online": "_online", "Offline": "_offline"}

    for upper, lower in trans.items():
        sub_class = sub_class.replace(upper, lower)

    return sub_class.lower()


for sub_name in globals().copy():
    if sub_name.endswith("Offline") or sub_name.endswith("Online"):
        cmd_name = class_to_tg(sub_name)
        m = getattr(this_module, sub_name)
        logging.info("Mapping %s to %s", cmd_name, m)
        vars()[cmd_name] = m

if __name__ == "__main__":
    sub = ZimuxiaOnline()
    search = sub.search_preview("最爱")
    print(search)
    uh = "4bcd33af9be2fba0060c388e984c7a2509e7e654"
    result = sub.search_result(uh)
    print(json.dumps(result, ensure_ascii=False))



File: yyetsbot/config.py

# coding: utf-8
# YYeTsBot - config.py
# 2019/8/15 18:42

__author__ = "Benny <benny.think@gmail.com>"

import os

# website config
# yyets
BASE_URL = "http://www.rrys2020.com"
LOGIN_URL = "http://www.rrys2020.com/user/login"
GET_USER = "http://www.rrys2020.com/user/login/getCurUserTopInfo"
YYETS_SEARCH_URL = "http://www.rrys2020.com/search?keyword={kw}&type=resource"
AJAX_LOGIN = "http://www.rrys2020.com/User/Login/ajaxLogin"
SHARE_URL = "http://www.rrys2020.com/resource/ushare"
SHARE_WEB = "http://got002.com/resource.html?code={code}"
# http://got002.com/api/v1/static/resource/detail?code=9YxN91
SHARE_API = "http://got002.com/api/v1/static/resource/detail?code={code}"

# fix zimuxia
FIX_RESOURCE = "http://www.zimuxia.cn/portfolio/{name}"
FIX_SEARCH = "http://www.zimuxia.cn/?s={kw}"

# zhuixinfan
ZHUIXINFAN_SEARCH = "http://www.fanxinzhui.com/list?k={}"
ZHUIXINFAN_RESOURCE = "http://www.fanxinzhui.com{}"
# yyets website
DOMAIN = "https://yyets.click/"
WORKERS = f"{DOMAIN}resource?id=" + "{}"
# https://yyets.click/discuss#6464d5b1b27861fa44647e7e
DISCUSS = f"{DOMAIN}discuss#" + "{}"
# new zmz
NEWZMZ_SEARCH = "https://newzmz.com/subres/index/getres.html?keyword={}"
NEWZMZ_RESOURCE = "https://ysfx.tv/view/{}"

# BD2020
BD2020_SEARCH = "https://v.bd2020.me/search.jspx?q={}"

# XL720
XL720_SEARCH = "https://www.xl720.com/?s={}"

# authentication config
TOKEN = os.getenv("TOKEN")

# network and server config
PROXY = os.getenv("PROXY")
REDIS = os.getenv("REDIS", "redis")
MONGO = os.getenv("MONGO", "mongo")

# other
MAINTAINER = os.getenv("OWNER")
REPORT = os.getenv("REPORT", False)
# This name must match class name, other wise this bot won't running.
FANSUB_ORDER: str = os.getenv("ORDER") or "YYeTsOffline,ZimuxiaOnline,NewzmzOnline,ZhuixinfanOnline,XL720,BD2020"



File: yyetsbot/utils.py

# coding: utf-8
# YYeTsBot - utils.py
# 2019/8/15 20:27

__author__ = 'Benny <benny.think@gmail.com>'

import pathlib

import redis

from config import REDIS

r = redis.StrictRedis(host=REDIS, decode_responses=True)

cookie_file = pathlib.Path(__file__).parent / 'data' / 'cookies.dump'


def save_error_dump(uid, err: str):
    r.set(uid, err)


def get_error_dump(uid) -> str:
    err = r.get(uid)
    r.delete(uid)
    if not err:
        err = ""
    return err


def redis_announcement(content="", op="get"):
    if op == "get":
        return r.get("announcement")
    elif op == "set":
        r.set("announcement", content)
    elif op == "del":
        r.delete("announcement")


def today_request(request_type: str):
    if r.exists("usage"):
        dict_data: dict = r.hgetall("usage")
        dict_data[request_type] = int(dict_data[request_type]) + 1
    else:
        data_format: dict = dict(total=0, invalid=0, answer=0, success=0, fail=0)
        data_format[request_type] += 1
        dict_data = data_format

    r.hset("usage", mapping=dict_data)


def reset_request():
    r.delete("usage")


def show_usage():
    m = "今天我已经服务了{total}次🤓，无效请求{invalid}😆，主人回复{answer}次🤨，成功请求{success}次😝，失败请求{fail}次🤣"
    if r.exists("usage"):
        dict_data: dict = r.hgetall("usage")
    else:
        dict_data: dict = dict(total=0, invalid=0, answer=0, success=0, fail=0)

    return m.format(**dict_data)



File: yyetsbot/yyetsbot.py

# coding: utf-8
# YYeTsBot - bot.py
# 2019/8/15 18:27

__author__ = "Benny <benny.think@gmail.com>"

import io
import json
import logging
import os
import re
import tempfile
import time
from urllib.parse import quote_plus

import requests
import telebot
import zhconv
from apscheduler.schedulers.background import BackgroundScheduler
from telebot import apihelper, types
from tgbot_ping import get_runtime

import fansub
from config import DOMAIN, FANSUB_ORDER, MAINTAINER, PROXY, REPORT, TOKEN, YYETS_SEARCH_URL
from utils import get_error_dump, redis_announcement, reset_request, save_error_dump, show_usage, today_request

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(filename)s [%(levelname)s]: %(message)s")
if PROXY:
    apihelper.proxy = {"https": PROXY}

bot = telebot.TeleBot(TOKEN, num_threads=100)
angry_count = 0


@bot.message_handler(commands=["start"], chat_types=["private"])
def send_welcome(message):
    bot.send_chat_action(message.chat.id, "typing")
    bot.send_message(
        message.chat.id,
        "欢迎使用，直接发送想要的剧集标题给我就可以了，不需要其他关键字，我会帮你搜索。\n\n"
        "仅私聊使用，群组功能已禁用。"
        f"目前搜索优先级 {FANSUB_ORDER}\n "
        f"另外，可以尝试使用一下 https://yyets.click/ 哦！",
        parse_mode="html",
        disable_web_page_preview=True,
    )


@bot.message_handler(commands=["help"], chat_types=["private"])
def send_help(message):
    bot.send_chat_action(message.chat.id, "typing")
    bot.send_message(
        message.chat.id,
        """机器人无法使用或者报错？从 /ping 里可以看到运行状态以及最新信息。
    同时，你可以使用如下方式寻求使用帮助和报告错误：\n
    1. @BennyThink
    2. <a href='https://github.com/BennyThink/YYeTsBot/issues'>Github issues</a>
    3. <a href='https://t.me/mikuri520'>Telegram Channel</a>""",
        parse_mode="html",
        disable_web_page_preview=True,
    )


@bot.message_handler(commands=["ping"], chat_types=["private"])
def send_ping(message):
    logging.info("Pong!")
    bot.send_chat_action(message.chat.id, "typing")

    info = get_runtime("botsrunner_yyets_1")

    usage = ""
    if str(message.chat.id) == MAINTAINER:
        usage = show_usage()
    announcement = redis_announcement() or ""
    if announcement:
        announcement = f"\n\n*公告：{announcement}*\n\n"
    bot.send_message(message.chat.id, f"{info}\n\n{usage}\n{announcement}", parse_mode="markdown")


@bot.message_handler(commands=["settings"], chat_types=["private"])
def settings(message):
    is_admin = str(message.chat.id) == MAINTAINER
    # 普通用户只可以查看，不可以设置。
    # 管理员可以查看可以设置
    if message.text != "/settings" and not is_admin:
        bot.send_message(message.chat.id, "此功能只允许管理员使用。请使用 /ping 和 /settings 查看相关信息")
        return

    # 删除公告，设置新公告
    if message.text != "/settings":
        date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        text = message.text.replace("/settings", f"{date}\t")
        logging.info("New announcement %s", text)
        redis_announcement(text, "set")
        setattr(message, "text", "/settings")
        settings(message)
        return

    announcement = redis_announcement()
    markup = types.InlineKeyboardMarkup()
    btn1 = types.InlineKeyboardButton("删除公告", callback_data="announcement")
    if is_admin and announcement:
        markup.add(btn1)

    bot.send_message(message.chat.id, f"目前公告：\n\n {announcement or '暂无公告'}", reply_markup=markup)


@bot.callback_query_handler(func=lambda call: re.findall(r"announcement(\S*)", call.data))
def delete_announcement(call):
    bot.send_chat_action(call.message.chat.id, "typing")
    redis_announcement(op="del")

    bot.edit_message_text(f"目前公告：\n\n {redis_announcement() or '暂无公告'}", call.message.chat.id, call.message.message_id)


@bot.message_handler(commands=["credits"], chat_types=["private"])
def send_credits(message):
    bot.send_chat_action(message.chat.id, "typing")
    bot.send_message(
        message.chat.id,
        """感谢字幕组的无私奉献！本机器人资源来源:\n
    <a href="http://www.zmz2019.com/">人人影视</a>
    <a href="http://cili001.com/">磁力下载站</a>
    <a href="http://www.zhuixinfan.com/main.php">追新番</a>
    <a href="https://www.zimuxia.cn/">FIX 字幕侠</a>
    """,
        parse_mode="html",
        disable_web_page_preview=True,
    )


for sub_name in dir(fansub):
    if sub_name.endswith("_offline") or sub_name.endswith("_online"):

        @bot.message_handler(commands=[sub_name], chat_types=["private"])
        def varies_fansub(message):
            bot.send_chat_action(message.chat.id, "typing")
            # /YYeTsOffline 逃避可耻 /YYeTsOffline
            tv_name: str = re.findall(r"/.*line\s*(\S*)", message.text)[0]
            class_name: str = re.findall(r"/(.*line)", message.text)[0]
            class_ = getattr(fansub, class_name)

            if not tv_name:
                bot.send_message(
                    message.chat.id, f"{class_.__name__}: 请附加你要搜索的剧集名称，如 `/{class_name} 逃避可耻`", parse_mode="markdown"
                )
                return

            else:
                setattr(message, "text", tv_name)
            base_send_search(message, class_())


def download_to_io(photo):
    logging.info("Initializing bytes io...")
    mem = io.BytesIO()
    file_id = photo[-1].file_id
    logging.info("Downloading photos...")
    file_info = bot.get_file(file_id)
    content = bot.download_file(file_info.file_path)
    mem.write(content)
    logging.info("Downloading complete.")
    return mem


def send_my_response(message):
    bot.send_chat_action(message.chat.id, "record_video_note")
    # I may also send picture
    photo = message.photo
    uid = message.reply_to_message.caption
    text = f"主人说：{message.text or message.caption or '啥也没说😯'}"
    if photo:
        bot.send_chat_action(message.chat.id, "typing")
        logging.info("Photo received from maintainer")
        mem = download_to_io(photo)
        mem.name = f"{uid}.jpg"
        r = bot.send_photo(uid, mem.getvalue(), caption=text)
    else:
        r = bot.send_message(uid, text)

    logging.info("Reply has been sent to %s with message id %s", uid, r.message_id)
    bot.reply_to(message, "回复已经发送给这位用户")
    fw = bot.forward_message(message.chat.id, uid, r.message_id)
    time.sleep(3)
    bot.delete_message(message.chat.id, fw.message_id)
    logging.info("Forward has been deleted.")


@bot.message_handler(content_types=["photo", "text"], chat_types=["private"])
def send_search(message):
    if str(message.chat.id) == os.getenv("SPECIAL_ID") and message.text == "❤️":
        bot.reply_to(message, "❤️")
    # normal ordered search
    if message.text in ("Voice Chat started", "Voice Chat ended"):
        logging.warning("This is really funny %s", message.text)
        return
    base_send_search(message)


@bot.message_handler(content_types=["document"], chat_types=["private"])
def ban_user(message):
    if str(message.chat.id) != MAINTAINER:
        return

    mem = io.BytesIO()
    file_id = message.document.file_id
    file_info = bot.get_file(file_id)
    content = bot.download_file(file_info.file_path)
    mem.write(content)
    user_list = mem.getvalue().decode("u8").split("\n")
    yy = fansub.YYeTsOffline()
    client = yy.mongo
    user_col = client["zimuzu"]["users"]
    comment_col = client["zimuzu"]["comment"]
    text = ""
    for line in user_list:
        user, reason = line.split(maxsplit=1)
        ban = {"disable": True, "reason": reason}
        user_col.update_one({"username": user}, {"$set": {"status": ban}})
        comment_col.delete_many({"username": user})
        status = f"{user} 已经被禁言，原因：{reason}\n"
        logging.info("Banning %s", status)
        text += status
    bot.reply_to(message, text)
    mem.close()


def base_send_search(message, instance=None):
    if instance is None:
        fan = fansub.FansubEntrance()
    else:
        fan = instance
    bot.send_chat_action(message.chat.id, "typing")

    today_request("total")
    if (
        message.reply_to_message
        and message.reply_to_message.document
        and message.reply_to_message.document.file_name.startswith("error")
        and str(message.chat.id) == MAINTAINER
    ):
        today_request("answer")
        send_my_response(message)
        return

    name = zhconv.convert(message.text, "zh-hans")
    logging.info("Receiving message: %s from user %s(%s)", name, message.chat.username, message.chat.id)
    if name is None:
        today_request("invalid")
        with open("warning.webp", "rb") as sti:
            bot.send_message(message.chat.id, "不要调戏我！我会报警的")
            bot.send_sticker(message.chat.id, sti)
        return

    result = fan.search_preview(name)

    markup = types.InlineKeyboardMarkup()

    source = result.get("class")
    result.pop("class")
    count, MAX, warning = 0, 20, ""
    for url_hash, detail in result.items():
        if count > MAX:
            warning = f"*结果太多啦，目前只显示前{MAX}个。关键词再精准一下吧！*\n\n"
            break
        btn = types.InlineKeyboardButton(detail["name"], callback_data="choose%s" % url_hash)
        markup.add(btn)
        count += 1

    if result:
        logging.info("🎉 Resource match.")
        today_request("success")
        bot.reply_to(
            message,
            f"{warning}呐🌹，一共%d个结果，选一个呀！来源：%s" % (len(result), source),
            reply_markup=markup,
            parse_mode="markdown",
        )
    else:
        logging.warning("⚠️️ Resource not found")
        today_request("fail")
        bot.send_chat_action(message.chat.id, "typing")

        encoded = quote_plus(name)
        bot.reply_to(
            message,
            f"没有找到你想要的信息，是不是你打了错别字，或者搜索了一些国产影视剧。🤪\n" f"还是你想调戏我哦🙅‍ 本小可爱拒绝被调戏️\n\n" "⚠️如果确定要我背锅，那么请使用 /help 来提交错误",
            disable_web_page_preview=True,
        )
        if REPORT:
            btn = types.InlineKeyboardButton("快来修复啦", callback_data="fix")
            markup.add(btn)
            bot.send_chat_action(message.chat.id, "upload_document")
            bot.send_message(
                message.chat.id,
                f"《{name}》😭\n大部分情况下机器人是好用的，不要怀疑我的代码质量.\n"
                f"如果你真的确定是机器人出问题了，那么点下面的按钮叫 @BennyThink 来修！\n"
                f"⚠️报错前请三思，不要乱点，确保这锅应该甩给我。否则我会很生气的😡小心被拉黑哦",
                reply_markup=markup,
            )
            content = f""" 报告者：{message.chat.first_name}{message.chat.last_name or ""}@{message.chat.username or ""}({message.chat.id})
                            问题发生时间：{time.strftime("%Y-%m-%data %H:%M:%S", time.localtime(message.date))}
                            请求内容：{name} 
                            请求URL：{YYETS_SEARCH_URL.format(kw=encoded)}\n\n
                            
                        """
            save_error_dump(message.chat.id, content)


def magic_recycle(fan, call, url_hash):
    if fan.redis.exists(url_hash):
        return False
    else:
        logging.info("👏 Wonderful magic!")
        bot.answer_callback_query(call.id, "小可爱使用魔法回收了你的搜索结果，你再搜索一次试试看嘛🥺", show_alert=True)
        bot.delete_message(call.message.chat.id, call.message.message_id)
        return True


@bot.callback_query_handler(func=lambda call: re.findall(r"choose(\S*)", call.data))
def choose_link(call):
    fan = fansub.FansubEntrance()
    bot.send_chat_action(call.message.chat.id, "typing")
    # call.data is url_hash, with sha1, http://www.rrys2020.com/resource/36588
    resource_url_hash = re.findall(r"choose(\S*)", call.data)[0]
    if magic_recycle(fan, call, resource_url_hash):
        return

    result = fan.search_result(resource_url_hash)
    with tempfile.NamedTemporaryFile(mode="wb+", prefix=result["cnname"].replace("/", " "), suffix=".txt") as tmp:
        bytes_data = json.dumps(result["all"], ensure_ascii=False, indent=4).encode("u8")
        tmp.write(bytes_data)
        tmp.flush()
        with open(tmp.name, "rb") as f:
            if result.get("type") == "resource":
                caption = "{}\n\n{}".format(result["cnname"], result["share"])
            else:
                caption = result["all"].replace(r"\n", "  ")
            bot.send_chat_action(call.message.chat.id, "upload_document")
            bot.send_document(call.message.chat.id, f, caption=caption)


@bot.callback_query_handler(func=lambda call: re.findall(r"approve", call.data))
def approve_spam(call):
    obj_id = re.findall(r"approve(\S*)", call.data)[0]
    data = {"obj_id": obj_id, "token": TOKEN}
    requests.post(f"{DOMAIN}api/admin/spam", json=data)
    bot.answer_callback_query(call.id, "Approved")
    bot.delete_message(call.message.chat.id, call.message.message_id)


@bot.callback_query_handler(func=lambda call: re.findall(r"ban", call.data))
def ban_spam(call):
    obj_id = re.findall(r"ban(\S*)", call.data)[0]
    data = {"obj_id": obj_id, "token": TOKEN}
    requests.delete(f"{DOMAIN}api/admin/spam", json=data)
    bot.answer_callback_query(call.id, "Banned")
    bot.delete_message(call.message.chat.id, call.message.message_id)


@bot.callback_query_handler(func=lambda call: re.findall(r"unwelcome(\d*)", call.data))
def send_unwelcome(call):
    # this will come from me only
    logging.warning("I'm so unhappy!")
    message = call.message
    bot.send_chat_action(message.chat.id, "typing")

    # angry_count = angry_count + 1
    global angry_count
    angry_count += 1
    uid = re.findall(r"unwelcome(\d*)", call.data)[0]

    if uid:
        text = "人人影视主要提供欧美日韩等海外资源，你的这个真没有🤷‍。\n" "<b>麻烦你先从自己身上找原因</b>，我又不是你的专属客服。\n" "不要再报告这种错误了🙄️，面倒な。😡"
        bot.send_message(uid, text, parse_mode="html")
        bot.reply_to(message, f"有生之日 生气次数：{angry_count}")


@bot.callback_query_handler(func=lambda call: call.data == "fix")
def report_error(call):
    logging.error("Reporting error to maintainer.")
    bot.send_chat_action(call.message.chat.id, "typing")
    error_content = get_error_dump(call.message.chat.id)
    if error_content == "":
        bot.answer_callback_query(call.id, "多次汇报重复的问题并不会加快处理速度。", show_alert=True)
        return

    text = f"人人影视机器人似乎出现了一些问题🤔🤔🤔……{error_content[0:300]}"

    markup = types.InlineKeyboardMarkup()
    btn = types.InlineKeyboardButton("unwelcome", callback_data=f"unwelcome{call.message.chat.id}")
    markup.add(btn)

    bot.send_message(MAINTAINER, text, disable_web_page_preview=True, reply_markup=markup)

    with tempfile.NamedTemporaryFile(mode="wb+", prefix=f"error_{call.message.chat.id}_", suffix=".txt") as tmp:
        tmp.write(error_content.encode("u8"))
        tmp.flush()

        with open(tmp.name, "rb") as f:
            bot.send_chat_action(call.message.chat.id, "upload_document")
            bot.send_document(MAINTAINER, f, caption=str(call.message.chat.id))

    bot.answer_callback_query(call.id, "Debug信息已经发送给维护者，请耐心等待回复~", show_alert=True)


if __name__ == "__main__":
    logging.info("YYeTs bot is running...")
    scheduler = BackgroundScheduler()
    scheduler.add_job(reset_request, "cron", hour=0, minute=0)
    scheduler.start()
    bot.polling()



File: yyets/__init__.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - __init__.py
# 9/21/21 18:09
#

__author__ = "Benny <benny.think@gmail.com>"

import logging

import requests

API = "https://yyets.click/api/resource?"

logging.basicConfig(
    level=logging.INFO,
    format="[%(asctime)s %(filename)s:%(lineno)d %(levelname).1s] %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)


class Resource:
    def __init__(self):
        self.enname = None
        self.cnname = None

    def __str__(self):
        return f"{self.cnname} - {self.enname}"


class YYeTs:
    def __init__(self, keyword: "str"):
        self.result = []
        self.keyword = keyword
        self.search_api = f"{API}keyword={self.keyword}"
        self.resource_api = f"{API}id=%s"
        self.search()

    def search(self):
        data = requests.get(self.search_api).json()
        for info in data["data"]:
            r = Resource()
            setattr(r, "list", self.fetch(info))
            for k, v in info.items():
                setattr(r, k, v)
            self.result.append(r)

    def fetch(self, info):
        rid = info["id"]
        url = self.resource_api % rid
        headers = {"Referer": url}
        logging.info("Fetching %s...%s", info["cnname"], url)
        return requests.get(url, headers=headers).json()["data"]["list"]

    def __str__(self):
        return f"{self.keyword} - {self.search_api}"


if __name__ == "__main__":
    ins = YYeTs("逃避可耻")
    for i in ins.result:
        print(i)



File: yyets/BagAndDrag/bag.py

#!/usr/local/bin/python3
# coding: utf-8

# BagAndDrag - bag.py
# 1/10/21 15:29
#

__author__ = "Benny <benny.think@gmail.com>"

import contextlib
import json
import logging
import os
import pickle
import sys
import time
import traceback

import pymysql
import requests

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(filename)s [%(levelname)s]: %(message)s')

COOKIES = os.path.join(os.path.dirname(__file__), 'cookies.dump')
USERNAME = os.environ.get("USERNAME") or "321"
PASSWORD = os.environ.get("PASSWORD") or "xa31sge"

GET_USER = "http://www.rrys2020.com/user/login/getCurUserTopInfo"
AJAX_LOGIN = "http://www.rrys2020.com/User/Login/ajaxLogin"
RESOURCE = "http://www.rrys2020.com/resource/{id}"
SHARE_URL = "http://www.rrys2020.com/resource/ushare"
# http://got002.com/api/v1/static/resource/detail?code=9YxN91
API_DATA = "http://got002.com/api/v1/static/resource/detail?code={code}"
ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"


def save_cookies(requests_cookiejar):
    with open(COOKIES, 'wb') as f:
        pickle.dump(requests_cookiejar, f)


def load_cookies():
    with contextlib.suppress(Exception):
        with open(COOKIES, 'rb') as f:
            return pickle.load(f)


def login():
    data = {"account": USERNAME, "password": PASSWORD, "remember": 1}
    logging.info("login in as %s", data)
    r = requests.post(AJAX_LOGIN, data=data, headers={"User-Agent": ua})
    resp = r.json()
    if resp.get('status') == 1:
        logging.info("Login success! %s", r.cookies)
        save_cookies(r.cookies)
    else:
        logging.error("Login failed! %s", resp)
        sys.exit(1)
    r.close()


def is_cookie_valid() -> bool:
    cookie = load_cookies()
    r = requests.get(GET_USER, cookies=cookie, headers={"User-Agent": ua})
    return r.json()['status'] == 1


def insert_db(data: dict):
    try:
        # INSERT INTO resource VALUE(id,url,name,expire,data)
        sql = "INSERT INTO resource VALUE(%s,%s,%s,%s,%s,%s)"
        con = pymysql.Connect(host="127.0.0.1", user="root", password="root", database="yyets", charset="utf8mb4")
        cur = con.cursor()
        info = data["data"]["info"]
        id = info["id"]
        url = RESOURCE.format(id=id)
        name = '{cnname}\n{enname}\n{alias}'.format(cnname=info["cnname"], enname=info["enname"],
                                                    alias=info["aliasname"])
        expire = info["expire"]
        date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(expire)))
        d = json.dumps(data, ensure_ascii=False, indent=2)

        cur.execute(sql, (id, url, name, expire, date, d))
        con.commit()
    except Exception as e:
        logging.error("insert error %s", e)
        logging.error(traceback.format_exc())


def insert_error(rid, tb):
    logging.warning("Logging error into database %s", rid)
    sql = "INSERT INTO failure VALUE (%s,%s)"
    con = pymysql.Connect(host="127.0.0.1", user="root", password="root", database="yyets", charset="utf8mb4")
    cur = con.cursor()
    cur.execute(sql, (rid, tb))
    con.commit()


def __load_sample():
    with open("sample.json") as f:
        return json.load(f)


if __name__ == '__main__':
    d = __load_sample()
    insert_db(d)
    insert_error(2331, "eeeeee")



File: yyets/BagAndDrag/cfkv.py

#!/usr/local/bin/python3
# coding: utf-8

# BagAndDrag - cfkv.py
# 1/17/21 12:08
#

__author__ = "Benny <benny.think@gmail.com>"

import json
import os

import pymysql

con = pymysql.Connect(host="127.0.0.1", user="root", password="root", charset="utf8mb4", database="yyets",
                      cursorclass=pymysql.cursors.DictCursor)
cur = con.cursor()

SIZE = 3000
cur.execute("select count(id) from resource")
count = cur.fetchall()[0]["count(id)"]
LIMIT = count // SIZE + 1


def convert_kv():
    for i in range(1, LIMIT + 1):
        SQL = "select id,data from resource limit %d offset %d" % (SIZE, (i - 1) * SIZE)
        print(SQL)
        cur = con.cursor()
        cur.execute(SQL)
        data = cur.fetchall()
        write_data = []
        for datum in data:
            write_data.append({
                "key": str(datum["id"]),  # keys need to be str
                "value": datum['data']}
            )
        with open(f"kv/kv_data{i - 1}.json", "w") as f:
            json.dump(write_data, f, ensure_ascii=False)


def verify_kv_data():
    files = os.listdir("kv")
    rows = 0
    for file in files:
        if file.startswith("kv_data"):
            with open(f"kv/{file}") as f:
                data = json.load(f)
                rows += len(data)
    print(rows, count)
    # assert rows == count


def dump_index():
    cur = con.cursor()
    indexes = {}
    cur.execute("select name, id from resource")
    data = cur.fetchall()
    for datum in data:
        name = datum["name"]
        rid = datum["id"]
        indexes[name] = rid
    with open("kv/index.json", "w") as f:
        write_data = [
            {
                "key": "index",
                "value": json.dumps(indexes, ensure_ascii=False)
            }
        ]
        json.dump(write_data, f, ensure_ascii=False, indent=2)


def generate_command():
    files = os.listdir("kv")
    tpl = "wrangler kv:bulk put --namespace-id=01d666b5ebae464193998bb074f672cf {filename}"
    shell = []
    for file in files:
        if file.endswith(".json"):
            shell.append(tpl.format(filename=file) + "\n")
    with open("kv/bulk.sh", "w") as f:
        f.writelines(shell)


if __name__ == '__main__':
    convert_kv()
    verify_kv_data()
    dump_index()
    generate_command()



File: yyets/BagAndDrag/convert_db.py

#!/usr/local/bin/python3
# coding: utf-8

# BagAndDrag - convert_db.py
# 1/12/21 18:24
#

__author__ = "Benny <benny.think@gmail.com>"

# convert to mongodb and con_sqlite

import json
import sqlite3
from typing import List

import pymongo
import pymysql
import tqdm

con_mysql = pymysql.Connect(host="127.0.0.1", user="root", password="root", charset="utf8mb4", database="yyets",
                            cursorclass=pymysql.cursors.DictCursor
                            )

mongo_client = pymongo.MongoClient()
con_sqlite = sqlite3.connect("yyets.db")

SIZE = 2000


def create_sqlite_database():
    sql = ["""
   DROP TABLE IF EXISTS resource;
    """,
           """
         create table resource
           (
               id         int primary key,
               url        varchar(255) null unique ,
               name       text         null,
               expire     int          null,
               expire_cst varchar(255) null,
               data       longtext     null
       
           )
           """
           ]
    cur = con_sqlite.cursor()
    for s in sql:
        cur.execute(s)
    con_sqlite.commit()


def clear_mongodb():
    mongo_client.drop_database("yyets")


def sqlite_insert(data: List[dict]):
    cur = con_sqlite.cursor()
    sql = "INSERT INTO resource VALUES(?,?,?,?,?,?)"

    cur.executemany(sql, [list(i.values()) for i in data])
    con_sqlite.commit()


def mongodb_insert(data: List[dict]):
    db = mongo_client["yyets"]
    col = db["resource"]
    # deserialize data.data
    inserted = []
    for i in data:
        i["data"] = json.loads(i["data"])
        inserted.append(i)
    col.insert_many(inserted)


def main():
    create_sqlite_database()
    clear_mongodb()

    mysql_cur = con_mysql.cursor()
    mysql_cur.execute("select count(id) from resource")
    count = mysql_cur.fetchall()[0]["count(id)"]

    mysql_cur.execute("SELECT * FROM resource")

    with tqdm.tqdm(total=count * 2) as pbar:
        while True:
            data = mysql_cur.fetchmany(SIZE)
            if data:
                sqlite_insert(data)
                pbar.update(SIZE)
                mongodb_insert(data)
                pbar.update(SIZE)
            else:
                break


if __name__ == '__main__':
    main()
    con_mysql.close()
    con_sqlite.close()
    mongo_client.close()



File: yyets/BagAndDrag/drag.py

#!/usr/local/bin/python3
# coding: utf-8

# BagAndDrag - drag.py
# 1/10/21 15:38
#

__author__ = "Benny <benny.think@gmail.com>"

import argparse
import logging
import time
import traceback
from concurrent.futures import ThreadPoolExecutor

import requests
from tqdm import tqdm

from bag import (API_DATA, SHARE_URL, insert_db, insert_error, is_cookie_valid,
                 load_cookies, login)

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(filename)s [%(levelname)s]: %(message)s')
s = requests.session()
ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
s.headers.update({"User-Agent": ua})

parser = argparse.ArgumentParser()
# start_id, end_id, interval, concurrency
parser.add_argument("-s", help="Start id", type=int, default=1)
parser.add_argument("-e", help="End id", type=int, default=10)
parser.add_argument("-i", help="Interval", default=5, type=int)
parser.add_argument("-c", help="Concurrency", default=2, type=int)
args = parser.parse_args()

executor = ThreadPoolExecutor(max_workers=args.c)


def get_api_json(resource_id):
    try:
        time.sleep(args.i)
        if not is_cookie_valid():
            login()
        logging.info("resource id is %s", resource_id)
        res = s.post(SHARE_URL, data={"rid": resource_id}, cookies=load_cookies()).json()
        share_code = res['data'].split('/')[-1]
        logging.info("Share code is %s", share_code)
        data = s.get(API_DATA.format(code=share_code)).json()

        insert_db(data)
    except Exception:
        insert_error(resource_id, traceback.format_exc())


def main():
    total = args.e + 1 - args.s
    list(tqdm(executor.map(get_api_json, range(args.s, args.e + 1)), total=total))


if __name__ == '__main__':
    main()



File: yyets/BagAndDrag/create_db.py

#!/usr/local/bin/python3
# coding: utf-8

# BagAndDrag - create_db.py
# 1/10/21 15:23
#

__author__ = "Benny <benny.think@gmail.com>"

import pymysql

con = pymysql.Connect(host="127.0.0.1", user="root", password="root", charset="utf8mb4")

sql = [
    "CREATE DATABASE yyets CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;",
    "use yyets",
    """
    create table resource
    (
        id         int primary key,
        url        varchar(255) null unique ,
        name       text         null,
        expire     int          null,
        expire_cst varchar(255) null,
        data       longtext     null
    
    )charset utf8mb4;
    
    
    """,

    """
    create table failure
    (
        id        int primary key not null,
        traceback longtext        null
    )charset utf8mb4;
    """,

]
cur = con.cursor()
for s in sql:
    cur.execute(s)
con.close()



File: yyets/BagAndDrag/zimuxia/convert_db.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - convert_db.py
# 2/5/21 13:46
#

__author__ = "Benny <benny.think@gmail.com>"

# convert to mongodb and con_sqlite

import pymongo
import pymysql
import tqdm
import json

from typing import List

con_mysql = pymysql.Connect(host="127.0.0.1", user="root", password="root", charset="utf8mb4", database="zimuxia",
                            cursorclass=pymysql.cursors.DictCursor
                            )

mongo_client = pymongo.MongoClient()

SIZE = 2000


def clear_mongodb():
    mongo_client.drop_database("zimuxia")


def clear_mysql():
    con_mysql.cursor().execute("truncate table resource;")
    con_mysql.commit()


def mysql_insert(data: List[dict]):
    sql = "INSERT INTO resource VALUES(NULL,%(url)s,%(name)s,NULL,NULL,%(data)s)"
    cur = con_mysql.cursor()
    for i in data:
        cur.execute(sql, i)
    con_mysql.commit()


def mongodb_insert(data: List[dict]):
    db = mongo_client["zimuxia"]
    col = db["resource"]
    col.insert_many(data)


def main():
    clear_mongodb()
    clear_mysql()
    with open("result.json") as f:
        data = json.load(f)
    # [{"url": "https://www.zimuxia.cn/portfolio/%e6%888b%e5%8f%8b", "name": "我家的女儿交不到男朋友", "data":""}]
    mysql_insert(data)
    mongodb_insert(data)


if __name__ == '__main__':
    main()
    con_mysql.close()
    mongo_client.close()



File: yyets/BagAndDrag/zimuxia/zimuxia.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - zimuxia.py
# 2/5/21 12:44
#

__author__ = "Benny <benny.think@gmail.com>"

import requests
import random
import time
import json
from bs4 import BeautifulSoup
from urllib.parse import quote, unquote
import tqdm
import logging

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(filename)s [%(levelname)s]: %(message)s')

list_url = "https://www.zimuxia.cn/%e6%88%91%e4%bb%ac%e7%9a%84%e4%bd%9c%e5%93%81?set={}"
ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"

s = requests.session()
s.headers.update({"User-Agent": ua})

data = []


def get_list():
    for index in tqdm.trange(1, 89):
        time.sleep(random.random())
        url = list_url.format(index)
        list_html = s.get(url).text
        get_episode(list_html)


def get_episode(html_text):
    soup = BeautifulSoup(html_text, 'html.parser')
    episodes = soup.find_all("div", class_="pg-item")

    for block in episodes:
        url = block.a['href']
        name = unquote(url).split("https://www.zimuxia.cn/portfolio/")[1]
        logging.info("fetching %s", name)
        t = {"url": url, "name": name, "data": s.get(url).text}
        data.append(t)


def write_json():
    with open("result.json", "w") as f:
        json.dump(data, f, ensure_ascii=False, indent=4)


if __name__ == '__main__':
    get_list()
    write_json()



File: yyets/healthcheck/restart_service.py

#!/usr/local/bin/python3
# coding: utf-8

# untitled - restart_service.py
# 9/18/21 11:54
#

__author__ = "Benny <benny.think@gmail.com>"

import logging.handlers
import subprocess

import requests

filename = "yyets_restart.log"
formatter = logging.Formatter("[%(asctime)s %(filename)s:%(lineno)d %(levelname).1s] %(message)s", "%Y-%m-%d %H:%M:%S")
handler = logging.handlers.RotatingFileHandler(filename, maxBytes=1024 * 1024 * 100)
handler.setFormatter(formatter)
logger = logging.getLogger()
logger.addHandler(handler)
logger.setLevel(logging.INFO)

URL = "https://yyets.click/api/top"
# URL = "https://www.baidu.com/"
req = requests.get(URL)

if req.status_code != 200:
    logger.error("error! %s", req)
    cmd = "/usr/bin/docker-compose -f /home/WebsiteRunner/docker-compose.yml restart mongo yyets-web nginx"
    subprocess.check_output(cmd.split())
else:
    logger.info("YYeTs is running %s", req)



File: yyets/healthcheck/check.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - check.py
# 1/22/21 16:36
#

__author__ = "Benny <benny.think@gmail.com>"

import logging
import os

import requests
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from telethon import TelegramClient, events

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(filename)s [%(levelname)s]: %(message)s")
logging.getLogger("apscheduler.executors.default").propagate = False
api_id = int(os.environ.get("API_ID") or "3")
api_hash = os.environ.get("API_HASH") or "4"
bot_name = os.environ.get("BOT_NAME") or "yyets_bot"
bot_token = os.environ.get("BOT_token") or "123"

client = TelegramClient(
    "client-hc", api_id, api_hash, device_model="Benny-health-check", system_version="89", app_version="1.0.0"
)
check_status = []


@client.on(events.NewMessage(incoming=True, pattern="(?i).*欢迎使用，直接发送想要的剧集标题给我就可以了.*", from_users=bot_name))
async def my_event_handler(event):
    logging.info("Okay it's working %s", event)
    check_status.clear()


async def send_health_check():
    if check_status:
        # restart it
        await bot_warning()
    else:
        await client.send_message(bot_name, "/start")
        check_status.append("check")


async def bot_warning():
    logging.warning("Bot seems to be down. Restarting now....")
    message = "Bot is down!!!"
    url = f"https://api.telegram.org/bot{bot_token}/sendMessage?chat_id=260260121&text={message}"
    resp = requests.get(url).json()
    logging.warning(resp)


async def website_check():
    home = "https://yyets.click/"
    top = "https://yyets.click/api/top"
    message = ""
    try:
        resp1 = requests.get(home)
        resp2 = requests.get(top)
    except Exception as e:
        message += f"Website is down. Requests error:{e}\n"
        resp1 = resp2 = ""

    if getattr(resp1, "status_code", 0) != 200:
        content = getattr(resp1, "content", None)
        message += f"Website home is down!!! {content}\n"
    if getattr(resp2, "status_code", 0) != 200:
        content = getattr(resp1, "content", None)
        message += f"Website top is down!!! {content}\n"
    if message:
        url = f"https://api.telegram.org/bot{bot_token}/sendMessage?chat_id=260260121&text={message}"
        resp = requests.get(url).json()
        logging.error(resp)
        logging.error(message)
    else:
        logging.info("It's working home: %s bytes; top: %s bytes", len(resp1.content), len(resp2.content))


if __name__ == "__main__":
    scheduler = AsyncIOScheduler()
    scheduler.add_job(send_health_check, "interval", seconds=300)
    scheduler.add_job(website_check, "interval", seconds=60)
    scheduler.start()
    client.start()
    client.run_until_disconnected()



File: yyets/management/ui.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - ui.py
# 2/8/21 17:55
#

__author__ = "Benny <benny.think@gmail.com>"

import json
import time

import PySimpleGUI as sg

# All the stuff inside your window.
channel_map = {
    "movie": "电影",
    "tv": "电视剧"
}

complete = {
    "status": 1,
    "info": "OK",
    "data": {
        "info": {},
        "list": [
            {
                "season_num": "1",
                "season_cn": "第一季",
                "items": {},
                "formats": [
                    "MP4"
                ]
            }
        ]
    }
}

dl = {
    "itemid": "",
    "episode": "0",
    "name": "",
    "size": "",
    "yyets_trans": 0,
    "dateline": str(int(time.time())),
    "files": [
        {
            "way": "1",
            "way_cn": "电驴",
            "address": "",
            "passwd": ""
        },
        {
            "way": "2",
            "way_cn": "磁力",
            "address": "",
            "passwd": ""
        },
        {
            "way": "9",
            "way_cn": "百度网盘",
            "address": "",
            "passwd": ""
        },
        {
            "way": "115",
            "way_cn": "115网盘",
            "address": "",
            "passwd": ""
        }
    ]
}
item_structure = {
    "MP4": [
    ]
}


def get_value():
    for i in range(1, int(episode_input[1].get()) + 1):
        d = dl.copy()
        d["episode"] = str(i)
        d["name"] = "{}第{}集".format(cn_input[1].get(), i)
        item_structure["MP4"].append(d)

    info_structure = {
        "id": 0,
        "cnname": cn_input[1].get(),
        "enname": en_input[1].get(),
        "aliasname": alias_input[1].get(),
        "channel": channel_input[1].get(),
        "channel_cn": channel_map.get(channel_input[1].get(), ""),
        "area": area_input[1].get(),
        "show_type": "",
        "expire": "1610401225",
        "views": 0
    }

    complete["data"]["info"] = info_structure
    complete["data"]["list"][0]["items"] = item_structure

    with open("sample.json", "w") as f:
        json.dump(complete, f, indent=4, ensure_ascii=False)


cn_input = [sg.Text('cn name'), sg.InputText()]
en_input = [sg.Text('en name'), sg.InputText()]
alias_input = [sg.Text('alias name'), sg.InputText()]
channel_input = [sg.Text('channel'), sg.Combo(['movie', 'tv'], "tv")]
area_input = [sg.Text('area'), sg.Combo(['美国', '日本', '韩国', '英国'], "美国")]
episode_input = [sg.Text('episode count'), sg.InputText()]

layout = [cn_input, en_input, alias_input, channel_input, area_input, episode_input,
          [sg.Button('Ok')]
          ]

# Create the Window
window = sg.Window('Management', layout)
# Event Loop to process "events" and get the "values" of the inputs
while True:
    event, values = window.read()
    if event == sg.WIN_CLOSED or event == 'Cancel':  # if user closes window or clicks cancel
        break
    if event == "Ok":
        print('You entered ', values[0], values[1], values[2])
        get_value()
        break
window.close()



File: yyetsweb/server.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - server.py
# 2/5/21 21:02
#

__author__ = "Benny <benny.think@gmail.com>"

import logging
import os
import pathlib
import threading
from zoneinfo import ZoneInfo

import tornado.autoreload
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from tornado import httpserver, ioloop, options, web
from tornado.log import enable_pretty_logging

from common.dump_db import entry_dump
from common.sync import YYSub, sync_douban
from common.utils import Cloudflare, setup_logger
from databases.base import SearchEngine
from databases.other import Other
from handlers.base import IndexHandler, NotFoundHandler
from handlers.comment import (
    CommentChildHandler,
    CommentHandler,
    CommentNewestHandler,
    CommentReactionHandler,
    CommentSearchHandler,
    NotificationHandler,
)
from handlers.douban import DoubanHandler, DoubanReportHandler
from handlers.grafana import (
    GrafanaIndexHandler,
    GrafanaQueryHandler,
    GrafanaSearchHandler,
    MetricsHandler,
)
from handlers.oauth import (
    FacebookAuth2LoginHandler,
    GitHubOAuth2LoginHandler,
    GoogleOAuth2LoginHandler,
    MSOAuth2LoginHandler,
    TwitterOAuth2LoginHandler,
)
from handlers.other import (
    AnnouncementHandler,
    BlacklistHandler,
    CaptchaHandler,
    CategoryHandler,
    DBDumpHandler,
    SpamProcessHandler,
)
from handlers.resources import (
    AdsenseStatusHandler,
    NameHandler,
    ResourceHandler,
    ResourceLatestHandler,
    TopHandler,
)
from handlers.user import LikeHandler, UserAvatarHandler, UserEmailHandler, UserHandler

enable_pretty_logging()
setup_logger()

if os.getenv("debug"):
    logging.getLogger().setLevel(logging.DEBUG)


class RunServer:
    static_path = pathlib.Path(__file__).parent.joinpath("templates")
    handlers = [
        (r"/", IndexHandler),
        (r"/api/resource", ResourceHandler),
        (r"/api/resource/latest", ResourceLatestHandler),
        (r"/api/top", TopHandler),
        (r"/api/like", LikeHandler),
        (r"/api/user", UserHandler),
        (r"/api/user/avatar/?(.*)", UserAvatarHandler),
        (r"/api/user/email", UserEmailHandler),
        (r"/api/name", NameHandler),
        (r"/api/adsense", AdsenseStatusHandler),
        (r"/api/comment", CommentHandler),
        (r"/api/comment/search", CommentSearchHandler),
        (r"/api/comment/reaction", CommentReactionHandler),
        (r"/api/comment/child", CommentChildHandler),
        (r"/api/comment/newest", CommentNewestHandler),
        (r"/api/captcha", CaptchaHandler),
        (r"/api/metrics", MetricsHandler),
        (r"/api/grafana/", GrafanaIndexHandler),
        (r"/api/grafana/search", GrafanaSearchHandler),
        (r"/api/grafana/query", GrafanaQueryHandler),
        (r"/api/blacklist", BlacklistHandler),
        (r"/api/db_dump", DBDumpHandler),
        (r"/api/announcement", AnnouncementHandler),
        (r"/api/douban", DoubanHandler),
        (r"/api/douban/report", DoubanReportHandler),
        (r"/api/notification", NotificationHandler),
        (r"/api/category", CategoryHandler),
        (r"/api/admin/spam", SpamProcessHandler),
        (r"/auth/github", GitHubOAuth2LoginHandler),
        (r"/auth/google", GoogleOAuth2LoginHandler),
        (r"/auth/twitter", TwitterOAuth2LoginHandler),
        (r"/auth/microsoft", MSOAuth2LoginHandler),
        (r"/auth/facebook", FacebookAuth2LoginHandler),
        (
            r"/(.*\.html|.*\.js|.*\.css|.*\.png|.*\.jpg|.*\.ico|.*\.gif|.*\.woff2|.*\.gz|.*\.zip|"
            r".*\.svg|.*\.json|.*\.txt)",
            web.StaticFileHandler,
            {"path": static_path},
        ),
    ]
    settings = {
        "cookie_secret": os.getenv("cookie_secret", "eo2kcgpKwXj8Q3PKYj6nIL1J4j3b58DX"),
        "default_handler_class": NotFoundHandler,
        "login_url": "/login",
        "google_oauth": {
            "key": os.getenv("GOOGLE_CLIENT_ID"),
            "secret": os.getenv("GOOGLE_CLIENT_SECRET"),
        },
        "github_oauth": {
            "key": os.getenv("GITHUB_CLIENT_ID"),
            "secret": os.getenv("GITHUB_CLIENT_SECRET"),
        },
        "ms_oauth": {
            "key": os.getenv("MS_CLIENT_ID"),
            "secret": os.getenv("MS_CLIENT_SECRET"),
        },
        "fb_oauth": {
            "key": os.getenv("FB_CLIENT_ID"),
            "secret": os.getenv("FB_CLIENT_SECRET"),
        },
        "twitter_consumer_key": os.getenv("TWITTER_CONSUMER_KEY"),
        "twitter_consumer_secret": os.getenv("TWITTER_CONSUMER_SECRET"),
    }
    application = web.Application(handlers, **settings)

    @staticmethod
    def run_server(port, host):
        tornado_server = httpserver.HTTPServer(RunServer.application, xheaders=True)
        tornado_server.bind(port, host)
        if os.getenv("PYTHON_DEV"):
            tornado_server.start(1)
            tornado.autoreload.start()
        else:
            tornado_server.start(0)

        try:
            print("Server is running on http://{}:{}".format(host, port))
            ioloop.IOLoop.instance().current().start()
        except KeyboardInterrupt:
            ioloop.IOLoop.instance().stop()
            print('"Ctrl+C" received, exiting.\n')


if __name__ == "__main__":
    timez = ZoneInfo("Asia/Shanghai")
    scheduler = BackgroundScheduler(timezone=timez)
    scheduler.add_job(Other().reset_top, trigger=CronTrigger.from_crontab("0 0 1 * *"))
    scheduler.add_job(sync_douban, trigger=CronTrigger.from_crontab("1 1 1 * *"))
    scheduler.add_job(entry_dump, trigger=CronTrigger.from_crontab("2 2 1 * *"))
    scheduler.add_job(Other().import_ban_user, "interval", seconds=300)
    scheduler.add_job(Other().fill_user_hash, "interval", seconds=60)
    scheduler.add_job(Cloudflare().clear_fw, trigger=CronTrigger.from_crontab("0 0 */3 * *"))
    scheduler.add_job(YYSub().run, trigger=CronTrigger.from_crontab("0 1 * * *"))

    scheduler.start()
    logging.info("Dumping database and ingesting data for Meilisearh...")
    if not os.getenv("PYTHON_DEV"):
        threading.Thread(target=entry_dump).start()
    # meilisearch tasks
    if os.getenv("MEILISEARCH"):
        logging.info("%s Searching with Meilisearch. %s", "#" * 10, "#" * 10)
        engine = SearchEngine()
        threading.Thread(target=engine.run_import).start()
        threading.Thread(target=engine.monitor_yyets).start()
        threading.Thread(target=engine.monitor_douban).start()
        threading.Thread(target=engine.monitor_comment).start()

    options.define("p", default=8888, help="running port", type=int)
    options.define("h", default="127.0.0.1", help="listen address", type=str)
    options.parse_command_line()
    p = options.options.p
    h = options.options.h
    banner = """
    ▌ ▌ ▌ ▌     ▀▛▘
    ▝▞  ▝▞  ▞▀▖  ▌  ▞▀▘
     ▌   ▌  ▛▀   ▌  ▝▀▖
     ▘   ▘  ▝▀▘  ▘  ▀▀ 
                        
     Lazarus came back from the dead. By @BennyThink
                """
    print(banner)
    RunServer.run_server(port=p, host=h)



File: yyetsweb/databases/user.py

#!/usr/bin/env python3
# coding: utf-8
import os
import random
import re
from hashlib import md5, sha256
from http import HTTPStatus

import filetype
import pymongo
import requests
from passlib.handlers.pbkdf2 import pbkdf2_sha256

from common.utils import send_mail, ts_date
from databases.base import Mongo, Redis


class Like(Mongo):
    projection = {"_id": False, "data.info": True}

    def get_user_like(self, username: str) -> list:
        like_list = self.db["users"].find_one({"username": username}).get("like", [])
        data = (
            self.db["yyets"]
            .find({"data.info.id": {"$in": like_list}}, self.projection)
            .sort("data.info.views", pymongo.DESCENDING)
        )
        return list(data)

    def add_remove_fav(self, resource_id: int, username: str) -> dict:
        returned = {"status_code": 0, "message": ""}
        like_list: list = self.db["users"].find_one({"username": username}).get("like", [])
        if resource_id in like_list:
            returned["status_code"] = HTTPStatus.OK
            returned["message"] = "已取消收藏"
            like_list.remove(resource_id)
        else:
            returned["status_code"] = HTTPStatus.CREATED
            returned["message"] = "已添加收藏"
            like_list.append(resource_id)

        value = dict(like=like_list)
        self.db["users"].update_one({"username": username}, {"$set": value})
        return returned


class User(Mongo, Redis):
    def login_user(
        self,
        username: str,
        password: str,
        captcha: str,
        captcha_id: str,
        ip: str,
        browser: str,
    ) -> dict:
        # verify captcha in the first place.
        correct_captcha = self.r.get(captcha_id)
        if correct_captcha is None:
            return {
                "status_code": HTTPStatus.BAD_REQUEST,
                "message": "验证码已过期",
                "status": False,
            }
        elif correct_captcha.lower() == captcha.lower():
            self.r.expire(captcha_id, 0)
        else:
            return {
                "status_code": HTTPStatus.FORBIDDEN,
                "message": "验证码错误",
                "status": False,
            }
        # check user account is locked.

        data = self.db["users"].find_one({"username": username}) or {}
        if data.get("status", {}).get("disable"):
            return {
                "status_code": HTTPStatus.FORBIDDEN,
                "status": False,
                "message": data.get("status", {}).get("reason"),
            }

        returned_value = {"status_code": 0, "message": ""}

        if data:
            stored_password = data["password"]
            if pbkdf2_sha256.verify(password, stored_password):
                returned_value["status_code"] = HTTPStatus.OK
            else:
                returned_value["status_code"] = HTTPStatus.FORBIDDEN
                returned_value["message"] = "用户名或密码错误"

        else:
            if os.getenv("DISABLE_REGISTER"):
                return {"status_code": HTTPStatus.BAD_REQUEST, "message": "本站已经暂停注册"}
            # register
            hash_value = pbkdf2_sha256.hash(password)
            try:
                self.db["users"].insert_one(
                    dict(
                        username=username,
                        password=hash_value,
                        date=ts_date(),
                        ip=ip,
                        browser=browser,
                        hash=sha256(username.encode("u8")).hexdigest(),
                    )
                )
                returned_value["status_code"] = HTTPStatus.CREATED
            except Exception as e:
                returned_value["status_code"] = HTTPStatus.INTERNAL_SERVER_ERROR
                returned_value["message"] = str(e)

        returned_value["username"] = data.get("username")
        returned_value["group"] = data.get("group", ["user"])
        return returned_value

    def get_user_info(self, username: str) -> dict:
        projection = {"_id": False, "password": False}
        data = self.db["users"].find_one({"username": username}, projection)
        data.update(group=data.get("group", ["user"]))
        data["hasAvatar"] = bool(data.pop("avatar", None))
        return data

    def update_user_last(self, username: str, now_ip: str) -> None:
        self.db["users"].update_one(
            {"username": username},
            {"$set": {"lastDate": (ts_date()), "lastIP": now_ip}},
        )

    def update_user_info(self, username: str, data: dict) -> dict:
        valid_fields = ["email"]
        valid_data = {}
        for field in valid_fields:
            if data.get(field):
                valid_data[field] = data[field]

        email_regex = r"@gmail\.com|@outlook\.com|@qq\.com|@163\.com"
        if valid_data.get("email") and not re.findall(email_regex, valid_data.get("email"), re.IGNORECASE):
            return {
                "status_code": HTTPStatus.BAD_REQUEST,
                "status": False,
                "message": "不支持的邮箱",
            }
        elif valid_data.get("email"):
            # rate limit
            user_email = valid_data.get("email")
            timeout_key = f"timeout-{username}"
            if self.r.get(timeout_key):
                return {
                    "status_code": HTTPStatus.TOO_MANY_REQUESTS,
                    "status": False,
                    "message": f"验证次数过多，请于{redis.ttl(timeout_key)}秒后尝试",
                }

            verify_code = random.randint(10000, 99999)
            valid_data["email"] = {"verified": False, "address": user_email}
            # send email confirm
            subject = "[人人影视下载分享站] 请验证你的邮箱"
            text = f"请输入如下验证码完成你的邮箱认证。验证码有效期为24小时。<br>" f"如果您未有此请求，请忽略此邮件。<br><br>验证码： {verify_code}"
            context = {"username": username, "text": text}
            send_mail(user_email, subject, context)
            # 发送成功才设置缓存
            self.r.set(timeout_key, username, ex=1800)
            self.r.hset(user_email, mapping={"code": verify_code, "wrong": 0})
            self.r.expire(user_email, 24 * 3600)

        self.db["users"].update_one({"username": username}, {"$set": valid_data})
        return {
            "status_code": HTTPStatus.CREATED,
            "status": True,
            "message": "邮件已经成功发送",
        }


class UserAvatar(User, Mongo):
    def add_avatar(self, username, avatar):
        self.db["users"].update_one({"username": username}, {"$set": {"avatar": avatar}})

        return {"status_code": HTTPStatus.CREATED, "message": "头像上传成功"}

    def get_avatar(self, username, user_hash=None):
        if user_hash:
            user = self.db["users"].find_one({"hash": user_hash})
        else:
            user = self.db["users"].find_one({"username": username})
        if user:
            img = user.get("avatar", b"")
            mime = filetype.guess_mime(img)
            return {"image": img, "content_type": mime}
        elif "@" in username:
            # fallback to gravatar
            url = f"https://gravatar.webp.se/avatar/{md5(username.encode('u8')).hexdigest()}"
            img = requests.get(url).content
            mime = filetype.guess_mime(img)
            return {"image": img, "content_type": mime}
        else:
            return {"image": None, "content_type": None}


class UserEmail(Mongo, Redis):
    def verify_email(self, username, code):
        email = self.db["users"].find_one({"username": username})["email"]["address"]
        verify_data = self.r.hgetall(email)
        wrong_count = int(verify_data["wrong"])
        MAX = 10
        if wrong_count >= MAX:
            self.db["users"].update_one(
                {"username": username},
                {"$set": {"status": {"disable": True, "reason": "verify email crack"}}},
            )
            return {
                "status": False,
                "status_code": HTTPStatus.FORBIDDEN,
                "message": "账户已被封锁",
            }
        correct_code = verify_data["code"]

        if correct_code == code:
            self.r.expire(email, 0)
            self.r.expire(f"timeout-{email}", 0)
            self.db["users"].update_one({"username": username}, {"$set": {"email.verified": True}})
            return {
                "status": True,
                "status_code": HTTPStatus.CREATED,
                "message": "邮箱已经验证成功",
            }
        else:
            self.r.hset(email, "wrong", wrong_count + 1)
            return {
                "status": False,
                "status_code": HTTPStatus.FORBIDDEN,
                "message": f"验证码不正确。你还可以尝试 {MAX - wrong_count} 次",
            }



File: yyetsweb/databases/comment.py

#!/usr/bin/env python3
# coding: utf-8
import contextlib
import os
import re
from http import HTTPStatus

import pymongo
from bson import ObjectId

from common.utils import check_spam, send_mail, ts_date
from databases.base import Mongo
from databases.other import Captcha, SpamProcess


class Comment(Mongo):
    def __init__(self):
        super().__init__()
        self.inner_page = 1
        self.inner_size = 5
        self.projection = {"ip": False, "parent_id": False}

    @staticmethod
    def convert_objectid(data):
        # change _id to id, remove _id
        for item in data:
            item["id"] = str(item["_id"])
            item.pop("_id")
            for child in item.get("children", []):
                with contextlib.suppress(Exception):
                    child["id"] = str(child["_id"])
                    child.pop("_id")

    def find_children(self, parent_data):
        for item in parent_data:
            children_ids = item.get("children", [])
            condition = {
                "_id": {"$in": children_ids},
                "deleted_at": {"$exists": False},
                "type": "child",
            }
            children_count = self.db["comment"].count_documents(condition)
            children_data = (
                self.db["comment"]
                .find(condition, self.projection)
                .sort("_id", pymongo.DESCENDING)
                .limit(self.inner_size)
                .skip((self.inner_page - 1) * self.inner_size)
            )
            children_data = list(children_data)
            self.get_user_group(children_data)
            self.add_reactions(children_data)

            item["children"] = []
            if children_data:
                item["children"].extend(children_data)
                item["childrenCount"] = children_count
            else:
                item["childrenCount"] = 0

    def get_user_group(self, data):
        whitelist = os.getenv("whitelist", "").split(",")
        for comment in data:
            username = comment["username"]
            user = self.db["users"].find_one({"username": username}) or {}
            group = user.get("group", ["user"])
            comment["group"] = group
            comment["hasAvatar"] = bool(user.get("avatar"))
            if username in whitelist:
                comment["group"].append("publisher")

    def add_reactions(self, data):
        for comment in data:
            cid = comment.get("id") or comment.get("_id")
            cid = str(cid)
            reactions = (
                self.db["reactions"].find_one({"comment_id": cid}, projection={"_id": False, "comment_id": False}) or {}
            )
            for verb, users in reactions.items():
                if users:
                    comment.setdefault("reactions", []).append({"verb": verb, "users": users})

    def get_comment(self, resource_id: int, page: int, size: int, **kwargs) -> dict:
        self.inner_page = kwargs.get("inner_page", 1)
        self.inner_size = kwargs.get("inner_size", 5)
        comment_id = kwargs.get("comment_id")
        sort = kwargs.get("sort")
        if sort == "newest":
            sort = pymongo.DESCENDING
        else:
            sort = pymongo.ASCENDING

        condition = {
            "resource_id": resource_id,
            "deleted_at": {"$exists": False},
            "type": {"$ne": "child"},
        }
        if comment_id:
            # 搜索某个评论id的结果
            condition = {
                "deleted_at": {"$exists": False},
                "$or": [
                    # 如果是子评论id，搜索子评论，会将整个父评论带出
                    {"children": {"$in": [ObjectId(comment_id)]}},
                    # 如果是父评论id，搜索父评论，并且排除子评论的记录
                    {"_id": ObjectId(comment_id), "type": {"$ne": "child"}},
                ],
            }

        count = self.db["comment"].count_documents(condition)
        data = (
            self.db["comment"]
            .find(condition, self.projection)
            .sort("_id", pymongo.DESCENDING)
            .limit(size)
            .skip((page - 1) * size)
        ).sort("_id", sort)
        data = list(data)
        self.find_children(data)
        self.convert_objectid(data)
        self.get_user_group(data)
        self.add_reactions(data)

        return {"data": data, "count": count, "resource_id": resource_id}

    def add_comment(
        self,
        captcha: str,
        captcha_id: int,
        content: str,
        resource_id: int,
        ip: str,
        username: str,
        browser: str,
        parent_comment_id=None,
    ) -> dict:
        user_data = self.db["users"].find_one({"username": username})
        # old user is allowed to comment without verification
        if not self.is_old_user(username) and user_data.get("email", {}).get("verified", False) is False:
            return {
                "status_code": HTTPStatus.TEMPORARY_REDIRECT,
                "message": "你需要验证邮箱才能评论，请到个人中心进行验证",
            }
        returned = {"status_code": 0, "message": ""}
        # check if this user is blocked
        reason = self.is_user_blocked(username)
        if reason:
            return {"status_code": HTTPStatus.FORBIDDEN, "message": reason}
        if check_spam(ip, browser, username, content) != 0:
            document = {
                "username": username,
                "ip": ip,
                "date": ts_date(),
                "browser": browser,
                "content": content,
                "resource_id": resource_id,
            }
            inserted_id = self.db["spam"].insert_one(document).inserted_id
            document["_id"] = str(inserted_id)
            SpamProcess.request_approval(document)
            return {
                "status_code": HTTPStatus.FORBIDDEN,
                "message": f"possible spam, reference id: {inserted_id}",
            }

        user_group = user_data.get("group", [])
        if not user_group:
            # admin don't have to verify code
            verify_result = Captcha().verify_code(captcha, captcha_id)
            if os.getenv("PYTHON_DEV"):
                pass
            elif not verify_result["status"]:
                returned["status_code"] = HTTPStatus.BAD_REQUEST
                returned["message"] = verify_result["message"]
                return returned

        exists = self.db["yyets"].find_one({"data.info.id": resource_id})
        if not exists:
            returned["status_code"] = HTTPStatus.NOT_FOUND
            returned["message"] = "资源不存在"
            return returned

        if parent_comment_id:
            exists = self.db["comment"].find_one({"_id": ObjectId(parent_comment_id)})
            if not exists:
                returned["status_code"] = HTTPStatus.NOT_FOUND
                returned["message"] = "评论不存在"
                return returned

        basic_comment = {
            "username": username,
            "ip": ip,
            "date": ts_date(),
            "browser": browser,
            "content": content,
            "resource_id": resource_id,
        }
        if parent_comment_id is None:
            basic_comment["type"] = "parent"
        else:
            basic_comment["type"] = "child"
        # 无论什么评论，都要插入一个新的document
        inserted_id: str = self.db["comment"].insert_one(basic_comment).inserted_id

        if parent_comment_id is not None:
            # 对父评论的子评论，需要给父评论加children id
            self.db["comment"].find_one_and_update(
                {"_id": ObjectId(parent_comment_id)},
                {"$push": {"children": inserted_id}},
            )
            self.db["comment"].find_one_and_update(
                {"_id": ObjectId(inserted_id)},
                {"$set": {"parent_id": ObjectId(parent_comment_id)}},
            )
        returned["status_code"] = HTTPStatus.CREATED
        returned["message"] = "评论成功"

        # notification
        if parent_comment_id:
            # find username

            self.db["notification"].find_one_and_update(
                {"username": exists["username"]},
                {"$push": {"unread": inserted_id}},
                upsert=True,
            )
            # send email
            parent_comment = self.db["comment"].find_one({"_id": ObjectId(parent_comment_id)})
            if resource_id == 233:
                link = f"https://yyets.click/discuss#{parent_comment_id}"
            else:
                link = f"https://yyets.click/resource?id={resource_id}#{parent_comment_id}"
            user_info = self.db["users"].find_one({"username": parent_comment["username"], "email.verified": True})
            if user_info:
                subject = "[人人影视下载分享站] 你的评论有了新的回复"
                pt_content = content.split("</reply>")[-1]
                text = (
                    f"你的评论 {parent_comment['content']} 有了新的回复：<br>{pt_content}"
                    f"<br>你可以<a href='{link}'>点此链接</a>查看<br><br>请勿回复此邮件"
                )
                context = {"username": username, "text": text}
                send_mail(user_info["email"]["address"], subject, context)
        return returned

    def delete_comment(self, comment_id):
        current_time = ts_date()
        count = (
            self.db["comment"]
            .update_one(
                {"_id": ObjectId(comment_id), "deleted_at": {"$exists": False}},
                {"$set": {"deleted_at": current_time}},
            )
            .modified_count
        )
        # 找到子评论，全部标记删除
        parent_data = self.db["comment"].find_one({"_id": ObjectId(comment_id)})
        if parent_data:
            child_ids = parent_data.get("children", [])
        else:
            child_ids = []
        count += (
            self.db["comment"]
            .update_many(
                {"_id": {"$in": child_ids}, "deleted_at": {"$exists": False}},
                {"$set": {"deleted_at": current_time}},
            )
            .modified_count
        )

        returned = {"status_code": 0, "message": "", "count": -1}
        if count == 0:
            returned["status_code"] = HTTPStatus.NOT_FOUND
            returned["count"] = 0
        else:
            returned["status_code"] = HTTPStatus.OK
            returned["count"] = count

        return returned


class CommentReaction(Mongo):
    def react_comment(self, username, data):
        # {"comment_id":"da23","😊":["user1","user2"]}
        comment_id = data["comment_id"]
        verb = data["verb"]
        method = data["method"]
        if not self.db["comment"].find_one({"_id": ObjectId(comment_id)}):
            return {
                "status": False,
                "message": "Where is your comments?",
                "status_code": HTTPStatus.NOT_FOUND,
            }

        if method == "POST":
            self.db["reactions"].update_one({"comment_id": comment_id}, {"$addToSet": {verb: username}}, upsert=True)
            code = HTTPStatus.CREATED
        elif method == "DELETE":
            self.db["reactions"].update_one({"comment_id": comment_id}, {"$pull": {verb: username}})
            code = HTTPStatus.ACCEPTED
        else:
            code = HTTPStatus.BAD_REQUEST
        return {"status": True, "message": "success", "status_code": code}


class CommentChild(Comment, Mongo):
    def __init__(self):
        super().__init__()
        self.page = 1
        self.size = 5
        self.projection = {"ip": False, "parent_id": False}

    def get_comment(self, parent_id: str, page: int, size: int) -> dict:
        condition = {
            "parent_id": ObjectId(parent_id),
            "deleted_at": {"$exists": False},
            "type": "child",
        }

        count = self.db["comment"].count_documents(condition)
        data = (
            self.db["comment"]
            .find(condition, self.projection)
            .sort("_id", pymongo.DESCENDING)
            .limit(size)
            .skip((page - 1) * size)
        )
        data = list(data)
        self.convert_objectid(data)
        self.get_user_group(data)
        return {
            "data": data,
            "count": count,
        }


class CommentNewest(Comment, Mongo):
    def __init__(self):
        super().__init__()
        self.page = 1
        self.size = 5
        self.projection = {"ip": False, "parent_id": False, "children": False}
        self.condition: "dict" = {"deleted_at": {"$exists": False}}

    def get_comment(self, page: int, size: int, keyword="") -> dict:
        # ID，时间，用户名，用户组，资源名，资源id
        count = self.db["comment"].count_documents(self.condition)
        data = (
            self.db["comment"]
            .find(self.condition, self.projection)
            .sort("_id", pymongo.DESCENDING)
            .limit(size)
            .skip((page - 1) * size)
        )
        data = list(data)
        self.convert_objectid(data)
        self.get_user_group(data)
        self.extra_info(data)
        return {
            "data": data,
            "count": count,
        }

    def extra_info(self, data):
        for i in data:
            resource_id = i.get("resource_id", 233)
            res = self.db["yyets"].find_one({"data.info.id": resource_id})
            if res:
                i["cnname"] = res["data"]["info"]["cnname"]


class CommentSearch(CommentNewest):
    def get_comment(self, page: int, size: int, keyword="") -> dict:
        self.projection.pop("children")
        self.condition.update(content={"$regex": f".*{keyword}.*", "$options": "i"})
        data = list(
            self.db["comment"]
            .find(self.condition, self.projection)
            .sort("_id", pymongo.DESCENDING)
            .limit(size)
            .skip((page - 1) * size)
        )
        self.convert_objectid(data)
        self.get_user_group(data)
        self.extra_info(data)
        self.fill_children(data)
        # final step - remove children
        for i in data:
            i.pop("children", None)
        return {
            "data": data,
        }

    def fill_children(self, data):
        for item in data:
            child_id: "list" = item.get("children", [])
            children = list(
                self.db["comment"].find({"_id": {"$in": child_id}}, self.projection).sort("_id", pymongo.DESCENDING)
            )
            self.convert_objectid(children)
            self.get_user_group(children)
            self.extra_info(children)
            data.extend(children)


class Notification(Mongo):
    def get_notification(self, username, page, size):
        # .sort("_id", pymongo.DESCENDING).limit(size).skip((page - 1) * size)
        notify = self.db["notification"].find_one({"username": username}, projection={"_id": False})
        if not notify:
            return {
                "username": username,
                "unread_item": [],
                "read_item": [],
                "unread_count": 0,
                "read_count": 0,
            }

        # size is shared
        unread = notify.get("unread", [])
        id_list = []
        for item in unread[(page - 1) * size : size * page]:
            id_list.append(item)
        notify["unread_item"] = self.get_content(id_list)

        size = size - len(unread)
        read = notify.get("read", [])
        id_list = []
        for item in read[(page - 1) * size : size * page]:
            id_list.append(item)
        notify["read_item"] = self.get_content(id_list)

        notify.pop("unread", None)
        notify.pop("read", None)
        notify["unread_count"] = len(unread)
        notify["read_count"] = len(read)
        return notify

    def get_content(self, id_list):
        comments = (
            self.db["comment"]
            .find({"_id": {"$in": id_list}}, projection={"ip": False, "parent_id": False})
            .sort("_id", pymongo.DESCENDING)
        )
        comments = list(comments)
        for comment in comments:
            comment["id"] = str(comment["_id"])
            comment.pop("_id")
            reply_to_id = re.findall(r'"(.*)"', comment["content"])[0]
            rtc = self.db["comment"].find_one(
                {"_id": ObjectId(reply_to_id)},
                projection={"content": True, "_id": False},
            )
            comment["reply_to_content"] = getattr(rtc, "content", "")

        return comments

    def update_notification(self, username, verb, comment_id):
        if verb == "read":
            v1, v2 = "read", "unread"
        else:
            v1, v2 = "unread", "read"
        self.db["notification"].find_one_and_update(
            {"username": username},
            {"$push": {v1: ObjectId(comment_id)}, "$pull": {v2: ObjectId(comment_id)}},
        )

        return {}



File: yyetsweb/databases/douban.py

#!/usr/bin/env python3
# coding: utf-8
import contextlib
import logging
import re
from http import HTTPStatus
from urllib.parse import unquote

import requests
from bs4 import BeautifulSoup
from retry import retry

from databases import DOUBAN_DETAIL, DOUBAN_SEARCH
from databases.base import Mongo
from databases.other import Captcha


class Douban(Mongo):
    def get_douban_data(self, rid: int) -> dict:
        with contextlib.suppress(Exception):
            return self.find_douban(rid)
        return {"posterData": None}

    def get_douban_image(self, rid: int) -> bytes:
        db_data = self.get_douban_data(rid)
        return db_data["posterData"]

    @retry(IndexError, tries=3, delay=5)
    def find_douban(self, resource_id: int):
        session = requests.Session()
        ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
        session.headers.update({"User-Agent": ua})

        douban_col = self.db["douban"]
        yyets_col = self.db["yyets"]
        data = douban_col.find_one(
            {"resourceId": resource_id}, {"_id": False, "raw": False}
        )
        if data:
            logging.info("Existing data for %s", resource_id)
            return data

        # data not found, craw from douban
        projection = {
            "data.info.cnname": True,
            "data.info.enname": True,
            "data.info.aliasname": True,
        }
        names = yyets_col.find_one({"data.info.id": resource_id}, projection=projection)
        if names is None:
            return {}
        cname = names["data"]["info"]["cnname"]
        logging.info("cnname for douban is %s", cname)

        search_html = session.get(DOUBAN_SEARCH.format(cname)).text
        logging.info("Analysis search html...length %s", len(search_html))
        soup = BeautifulSoup(search_html, "html.parser")
        douban_item = soup.find_all("div", class_="content")

        fwd_link = unquote(douban_item[0].a["href"])
        douban_id = re.findall(
            r"https://movie\.douban\.com/subject/(\d*)/.*", fwd_link
        )[0]
        final_data = self.get_craw_data(
            cname, douban_id, resource_id, search_html, session
        )
        douban_col.insert_one(final_data.copy())
        final_data.pop("raw")
        return final_data

    @staticmethod
    def get_craw_data(cname, douban_id, resource_id, search_html, session):
        detail_link = DOUBAN_DETAIL.format(douban_id)
        detail_html = session.get(detail_link).text
        logging.info("Analysis detail html...%s", detail_link)
        soup = BeautifulSoup(detail_html, "html.parser")

        directors = [i.text for i in (soup.find_all("a", rel="v:directedBy"))]
        release_date = (
            poster_image_link
        ) = rating = year_text = intro = writers = episode_count = episode_duration = ""
        with contextlib.suppress(IndexError):
            episode_duration = soup.find_all("span", property="v:runtime")[0].text
        for i in soup.find_all("span", class_="pl"):
            if i.text == "编剧":
                writers = re.sub(r"\s", "", list(i.next_siblings)[1].text).split("/")
            if i.text == "集数:":
                episode_count = str(i.nextSibling)
            if i.text == "单集片长:" and not episode_duration:
                episode_duration = str(i.nextSibling)
        actors = [i.text for i in soup.find_all("a", rel="v:starring")]
        genre = [i.text for i in soup.find_all("span", property="v:genre")]

        with contextlib.suppress(IndexError):
            release_date = soup.find_all("span", property="v:initialReleaseDate")[
                0
            ].text
        with contextlib.suppress(IndexError):
            poster_image_link = soup.find_all("div", id="mainpic")[0].a.img["src"]
        with contextlib.suppress(IndexError):
            rating = soup.find_all("strong", class_="ll rating_num")[0].text
        with contextlib.suppress(IndexError):
            year_text = re.sub(
                r"[()]", "", soup.find_all("span", class_="year")[0].text
            )
        with contextlib.suppress(IndexError):
            intro = re.sub(
                r"\s", "", soup.find_all("span", property="v:summary")[0].text
            )

        final_data = {
            "name": cname,
            "raw": {
                "search_url": DOUBAN_SEARCH.format(cname),
                "detail_url": detail_link,
                "search_html": search_html,
                "detail_html": detail_html,
            },
            "doubanId": int(douban_id),
            "doubanLink": detail_link,
            "posterLink": poster_image_link,
            "posterData": session.get(poster_image_link).content,
            "resourceId": resource_id,
            "rating": rating,
            "actors": actors,
            "directors": directors,
            "genre": genre,
            "releaseDate": release_date,
            "episodeCount": episode_count,
            "episodeDuration": episode_duration,
            "writers": writers,
            "year": year_text,
            "introduction": intro,
        }
        return final_data


class DoubanReport(Mongo):
    def get_error(self) -> dict:
        return dict(data=list(self.db["douban_error"].find(projection={"_id": False})))

    def report_error(
        self, captcha: str, captcha_id: int, content: str, resource_id: int
    ) -> dict:
        returned = {"status_code": 0, "message": ""}
        verify_result = Captcha().verify_code(captcha, captcha_id)
        if not verify_result["status"]:
            returned["status_code"] = HTTPStatus.BAD_REQUEST
            returned["message"] = verify_result["message"]
            return returned

        count = (
            self.db["douban_error"]
            .update_one(
                {"resource_id": resource_id},
                {"$push": {"content": content}},
                upsert=True,
            )
            .matched_count
        )
        return dict(count=count)



File: yyetsweb/databases/__init__.py

#!/usr/bin/env python3
# coding: utf-8

# YYeTsBot - __init__.py
# 2023-03-17  18:57

import logging
import os
import pathlib
import sys

import fakeredis
import pymongo
import redis

DOUBAN_SEARCH = "https://www.douban.com/search?cat=1002&q={}"
DOUBAN_DETAIL = "https://movie.douban.com/subject/{}/"

lib_path = pathlib.Path(__file__).parent.parent.parent.joinpath("yyetsbot").resolve().as_posix()

sys.path.append(lib_path)
from fansub import BD2020, XL720, NewzmzOnline, ZhuixinfanOnline, ZimuxiaOnline

logging.info(
    "Initialized: loading fansub...%s",
    (BD2020, XL720, NewzmzOnline, ZhuixinfanOnline, ZimuxiaOnline),
)

mongo_client = pymongo.MongoClient(
    host=os.getenv("MONGO", "localhost"),
    connect=True,
    connectTimeoutMS=5000,
    serverSelectionTimeoutMS=5000,
    maxPoolSize=300,
    minPoolSize=50,
    maxIdleTimeMS=600000,
)
db = mongo_client["zimuzu"]

try:
    redis_client = redis.StrictRedis(
        host=os.getenv("REDIS", "localhost"),
        decode_responses=True,
        max_connections=100,
    )
    redis_client.ping()
except redis.exceptions.ConnectionError:
    logging.warning("%s Using fakeredis now... %s", "#" * 10, "#" * 10)
    redis_client = fakeredis.FakeStrictRedis()



File: yyetsweb/databases/other.py

#!/usr/bin/env python3
# coding: utf-8
import base64
import json
import logging
import os
import random
import re
import string
import time
from hashlib import sha256

import pymongo
import requests
from bson import ObjectId
from captcha.image import ImageCaptcha

from common.utils import Cloudflare, ts_date
from databases.base import Mongo, Redis

cf = Cloudflare()

captcha_ex = 60 * 10
predefined_str = re.sub(r"[1l0oOI]", "", string.ascii_letters + string.digits)


class Announcement(Mongo):
    def get_announcement(self, page: int, size: int) -> dict:
        condition = {}
        count = self.db["announcement"].count_documents(condition)
        data = (
            self.db["announcement"]
            .find(condition, projection={"_id": True, "ip": False})
            .sort("_id", pymongo.DESCENDING)
            .limit(size)
            .skip((page - 1) * size)
        )
        data = list(data)
        for i in data:
            i["id"] = str(i["_id"])
            i.pop("_id")
        return {
            "data": data,
            "count": count,
        }

    def add_announcement(self, username, content, ip, browser):
        construct = {
            "username": username,
            "ip": ip,
            "date": ts_date(),
            "browser": browser,
            "content": content,
        }
        self.db["announcement"].insert_one(construct)


class Blacklist(Redis):
    def get_black_list(self):
        keys = self.r.keys("*")
        result = {}

        for key in keys:
            count = self.r.get(key)
            ttl = self.r.ttl(key)
            if ttl != -1:
                result[key] = dict(count=count, ttl=ttl)
        return result


class Category(Mongo):
    def get_category(self, query: dict):
        page, size, douban = query["page"], query["size"], query["douban"]
        query.pop("page")
        query.pop("size")
        query.pop("douban")
        query_dict = {}
        for key, value in query.items():
            query_dict[f"data.info.{key}"] = value
        logging.info("Query dict %s", query_dict)
        projection = {"_id": False, "data.list": False}
        data = self.db["yyets"].find(query_dict, projection=projection).limit(size).skip((page - 1) * size)
        count = self.db["yyets"].count_documents(query_dict)
        f = []
        for item in data:
            if douban:
                douban_data = self.db["douban"].find_one(
                    {"resourceId": item["data"]["info"]["id"]}, projection=projection
                )
                if douban_data:
                    douban_data["posterData"] = base64.b64encode(douban_data["posterData"]).decode("u8")
                    item["data"]["info"]["douban"] = douban_data
                else:
                    item["data"]["info"]["douban"] = {}
            f.append(item["data"]["info"])
        return dict(data=f, count=count)


class SpamProcess(Mongo):
    def ban_spam(self, obj_id: "str"):
        obj_id = ObjectId(obj_id)
        logging.info("Deleting spam %s", obj_id)
        spam = self.db["spam"].find_one({"_id": obj_id})
        username = spam["username"]
        self.db["spam"].delete_many({"username": username})
        # self.db["comment"].delete_many({"username": username})
        cf.ban_new_ip(spam["ip"])
        return {"status": True}

    def restore_spam(self, obj_id: "str"):
        obj_id = ObjectId(obj_id)
        spam = self.db["spam"].find_one({"_id": obj_id}, projection={"_id": False})
        logging.info("Restoring spam %s", spam)
        self.db["comment"].insert_one(spam)
        self.db["spam"].delete_one({"_id": obj_id})
        return {"status": True}

    @staticmethod
    def request_approval(document: "dict"):
        token = os.getenv("TOKEN")
        owner = os.getenv("OWNER")
        obj_id = document["_id"]
        data = {
            "text": json.dumps(document, ensure_ascii=False, indent=4),
            "chat_id": owner,
            "reply_markup": {
                "inline_keyboard": [
                    [
                        {"text": "approve", "callback_data": f"approve{obj_id}"},
                        {"text": "ban", "callback_data": f"ban{obj_id}"},
                    ]
                ]
            },
        }
        api = f"https://api.telegram.org/bot{token}/sendMessage"
        resp = requests.post(api, json=data).json()
        logging.info("Telegram response: %s", resp)


class Other(Mongo, Redis):
    def reset_top(self):
        # before resetting, save top data to history
        json_data = requests.get("http://127.0.0.1:8888/api/top").json()
        last_month = time.strftime("%Y-%m", time.localtime(time.time() - 3600 * 24))
        json_data["date"] = last_month
        json_data["type"] = "top"
        self.db["history"].insert_one(json_data)
        # save all the views data to history
        projection = {"_id": False, "data.info.views": True, "data.info.id": True}
        data = self.db["yyets"].find({}, projection).sort("data.info.views", pymongo.DESCENDING)
        result = {"date": last_month, "type": "detail"}
        for datum in data:
            rid = str(datum["data"]["info"]["id"])
            views = datum["data"]["info"]["views"]
            result[rid] = views
        self.db["history"].insert_one(result)
        # reset
        self.db["yyets"].update_many({}, {"$set": {"data.info.views": 0}})

    def import_ban_user(self):
        usernames = self.db["users"].find({"status.disable": True}, projection={"username": True})
        self.r.delete("user_blacklist")
        logging.info("Importing ban users to redis...%s", usernames)
        for username in [u["username"] for u in usernames]:
            self.r.hset("user_blacklist", username, 100)

    def fill_user_hash(self):
        users = self.db["users"].find({"hash": {"$exists": False}}, projection={"username": True})
        # do it old school
        for user in users:
            logging.info("Filling hash for %s", user)
            username = user["username"]
            hash_value = sha256(username.encode("u8")).hexdigest()
            self.db["users"].update_one({"username": username}, {"$set": {"hash": hash_value}})


class Captcha(Redis):
    def get_captcha(self, captcha_id):
        chars = "".join([random.choice(predefined_str) for _ in range(4)])
        image = ImageCaptcha()
        data = image.generate(chars)
        self.r.set(captcha_id, chars, ex=captcha_ex)
        return f"data:image/png;base64,{base64.b64encode(data.getvalue()).decode('ascii')}"

    def verify_code(self, user_input, captcha_id) -> dict:
        correct_code = self.r.get(captcha_id)
        if not correct_code:
            return {"status": False, "message": "验证码已过期"}
        if user_input.lower() == correct_code.lower():
            self.r.delete(correct_code)
            return {"status": True, "message": "验证通过"}
        else:
            return {"status": False, "message": "验证码错误"}



File: yyetsweb/databases/grafana.py

#!/usr/bin/env python3
# coding: utf-8
import time
from datetime import date, timedelta

import pymongo

from databases.base import Mongo


class GrafanaQuery(Mongo):
    def get_grafana_data(self, date_series) -> str:
        condition = {"date": {"$in": date_series}}
        projection = {"_id": False}
        return self.db["metrics"].find(condition, projection)


class Metrics(Mongo):
    def set_metrics(self, metrics_type: str):
        today = time.strftime("%Y-%m-%d", time.localtime())
        self.db["metrics"].update_one({"date": today}, {"$inc": {metrics_type: 1}}, upsert=True)

    def get_metrics(self, from_date: str, to_date: str) -> dict:
        start_int = [int(i) for i in from_date.split("-")]
        end_int = [int(i) for i in to_date.split("-")]
        sdate = date(*start_int)  # start date
        edate = date(*end_int)  # end date
        date_range = [str(sdate + timedelta(days=x)) for x in range((edate - sdate).days + 1)]
        condition = {"date": {"$in": date_range}}
        result = self.db["metrics"].find(condition, {"_id": False}).sort("date", pymongo.DESCENDING)

        return dict(metrics=list(result))



File: yyetsweb/databases/resources.py

#!/usr/bin/env python3
# coding: utf-8
import contextlib
import json
import logging
import os
import random

import pymongo
import requests
import zhconv
from tqdm import tqdm

from common.utils import hide_phone, ts_date
from databases.base import Mongo, Redis, SearchEngine
from databases.comment import CommentSearch


class Resource(SearchEngine):
    def fansub_search(self, class_name: str, kw: str):
        class_ = globals().get(class_name)
        result = class_().search_preview(kw)
        result.pop("class")
        if result:
            return list(result.values())
        else:
            return []

    def get_resource_data(self, resource_id: int, username: str) -> dict:
        data: "dict" = self.db["yyets"].find_one_and_update(
            {"data.info.id": resource_id},
            {"$inc": {"data.info.views": 1}},
            {"_id": False},
        )
        if not data:
            return {}
        if username:
            user_like_data = self.db["users"].find_one({"username": username})
            if user_like_data and resource_id in user_like_data.get("like", []):
                data["is_like"] = True
            else:
                data["is_like"] = False
        return data

    def search_resource(self, keyword: str, search_type: "str") -> dict:
        if os.getenv("MEILISEARCH"):
            return self.meili_search(keyword, search_type)
        else:
            return self.mongodb_search(keyword)

    def meili_search(self, keyword: "str", search_type: "str") -> dict:
        returned = {"data": [], "comment": [], "extra": []}
        if search_type == "default":
            yyets = self.search_yyets(keyword)
            comment = hide_phone(self.search_comment(keyword))
            returned["data"] = yyets
            returned["comment"] = comment
            return returned
        elif search_type == "douban":
            douban = self.search_douban(keyword)
            returned["data"] = douban
            return returned
        elif search_type == "fansub":
            # TODO disable fansub for now
            # fansub = self.search_extra(keyword)
            # returned["extra"] = fansub
            return returned
        else:
            return returned

    def mongodb_search(self, keyword: str) -> dict:
        # convert any text to zh-hans - only for traditional search with MongoDB
        keyword = zhconv.convert(keyword, "zh-hans")

        zimuzu_data = []
        returned = {"data": [], "extra": [], "comment": []}

        projection = {"_id": False, "data.info": True}

        resource_data = self.db["yyets"].find(
            {
                "$or": [
                    {"data.info.cnname": {"$regex": f".*{keyword}.*", "$options": "i"}},
                    {"data.info.enname": {"$regex": f".*{keyword}.*", "$options": "i"}},
                    {
                        "data.info.aliasname": {
                            "$regex": f".*{keyword}.*",
                            "$options": "i",
                        }
                    },
                ]
            },
            projection,
        )

        for item in resource_data:
            item["data"]["info"]["origin"] = "yyets"
            zimuzu_data.append(item["data"]["info"])

        # get comment
        c_search = []
        comments = CommentSearch().get_comment(1, 2**10, keyword)
        hide_phone(comments.get("data", []))
        for c in comments.get("data", []):
            comment_rid = c["resource_id"]
            res = self.db["yyets"].find_one({"data.info.id": comment_rid}, projection={"data.info": True})
            if res:
                c_search.append(
                    {
                        "username": c["username"],
                        "date": c["date"],
                        "comment": c["content"],
                        "commentID": c["id"],
                        "resourceID": comment_rid,
                        "resourceName": res["data"]["info"]["cnname"],
                        "origin": "comment",
                        "hasAvatar": c["hasAvatar"],
                        "hash": c.get("hash"),
                    }
                )
        # zimuzu -> comment -> extra
        if zimuzu_data:
            returned["data"] = zimuzu_data
        elif not c_search:
            # only returned when no data found
            returned["extra"] = self.search_extra(keyword)
        # comment data will always be returned
        returned["comment"] = c_search
        return returned

    def search_extra(self, keyword: "str") -> list:
        order = os.getenv("ORDER", "YYeTsOffline,ZimuxiaOnline,NewzmzOnline,ZhuixinfanOnline").split(",")
        order.pop(0)
        extra = []
        with contextlib.suppress(requests.exceptions.RequestException):
            for name in order:
                extra = self.fansub_search(name, keyword)
                if extra:
                    break
        return extra

    def patch_resource(self, new_data: dict):
        rid = new_data["resource_id"]
        new_data.pop("resource_id")
        old_data = self.db["yyets"].find_one(
            {"data.info.id": rid},
        )
        new_data["season_cn"] = self.convert_season(new_data["season_num"])
        # 1. totally empty resource:
        if len(old_data["data"]["list"]) == 0:
            new_data["season_cn"] = self.convert_season(new_data["season_num"])
            old_data["data"]["list"].append(new_data)
        else:
            for season in old_data["data"]["list"]:
                if new_data["season_num"] in [
                    season["season_num"],
                    int(season["season_num"]),
                ]:
                    user_format = new_data["formats"][0]
                    for u in new_data["items"][user_format]:
                        season["items"][user_format].append(u)

        self.db["yyets"].find_one_and_replace({"data.info.id": rid}, old_data)

    def add_resource(self, new_data: dict):
        rid = self.get_appropriate_id()
        new_data["data"]["info"]["id"] = rid
        self.db["yyets"].insert_one(new_data)
        return {"status": True, "message": "success", "id": rid}

    def delete_resource(self, data: dict):
        rid = data["resource_id"]
        meta = data.get("meta")
        if meta:
            db_data = self.db["yyets"].find_one({"data.info.id": rid})
            for season in db_data["data"]["list"]:
                for episode in season["items"].values():
                    for v in episode:
                        if (
                            v["episode"] == meta["episode"]
                            and v["name"] == meta["name"]
                            and v["size"] == meta["size"]
                            and v["dateline"] == meta["dateline"]
                        ):
                            episode.remove(v)
            # replace it
            self.db["yyets"].find_one_and_replace({"data.info.id": rid}, db_data)

        else:
            self.db["yyets"].delete_one({"data.info.id": rid})

    def get_appropriate_id(self):
        col = self.db["yyets"]
        random_id = random.randint(50000, 80000)
        data = col.find_one({"data.info.id": random_id}, projection={"_id": True})
        if data:
            return self.get_appropriate_id()
        else:
            return random_id

    @staticmethod
    def convert_season(number: [int, str]):
        if number in (0, "0"):
            return "正片"
        else:
            return f"第{number}季"


class Top(Mongo):
    projection = {"_id": False, "data.info": True}

    def get_most(self) -> list:
        projection = {"_id": False, "like": True}
        data = self.db["users"].find({}, projection)
        most_like = {}
        for item in data:
            for _id in item.get("like", []):
                most_like[_id] = most_like.get(_id, 0) + 1
        most = sorted(most_like, key=most_like.get)
        most.reverse()
        most_like_data = self.db["yyets"].find({"data.info.id": {"$in": most}}, self.projection).limit(15)
        return list(most_like_data)

    def get_top_resource(self) -> dict:
        area_dict = dict(ALL={"$regex": ".*"}, US="美国", JP="日本", KR="韩国", UK="英国")
        all_data = {"ALL": "全部"}
        for abbr, area in area_dict.items():
            data = (
                self.db["yyets"]
                .find(
                    {"data.info.area": area, "data.info.id": {"$ne": 233}},
                    self.projection,
                )
                .sort("data.info.views", pymongo.DESCENDING)
                .limit(15)
            )
            all_data[abbr] = list(data)

        all_data["class"] = area_dict
        return all_data


class ResourceLatest(Mongo, Redis):
    def get_latest_resource(self) -> dict:
        key = "latest-resource"
        latest = self.r.get(key)
        if latest:
            logging.info("Cache hit for latest resource")
            latest = json.loads(latest)
            latest["data"] = latest["data"][:100]
        else:
            logging.warning("Cache miss for latest resource")
            latest = ResourceLatest().query_db()
            self.r.set(key, json.dumps(latest, ensure_ascii=False))
        return latest

    def query_db(self) -> dict:
        col = self.db["yyets"]
        projection = {"_id": False, "status": False, "info": False}
        episode_data = {}
        for res in tqdm(col.find(projection=projection), total=col.count_documents({})):
            for season in res["data"].get("list", []):
                for item in season["items"].values():
                    for single in item:
                        ts = single["dateline"]
                        res_name = res["data"]["info"]["cnname"]
                        name = "{}-{}".format(res_name, single["name"])
                        size = single["size"]
                        episode_data[name] = {
                            "timestamp": ts,
                            "size": size,
                            "resource_id": res["data"]["info"]["id"],
                            "res_name": res_name,
                            "date": ts_date(int(ts)),
                        }

        sorted_res: list = sorted(episode_data.items(), key=lambda x: x[1]["timestamp"], reverse=True)
        limited_res = dict(sorted_res[:100])
        ok = []
        for k, v in limited_res.items():
            t = {"name": k}
            t.update(v)
            ok.append(t)
        return dict(data=ok)

    def refresh_latest_resource(self):
        logging.info("Getting new resources...")
        latest = self.query_db()
        self.r.set("latest-resource", json.dumps(latest, ensure_ascii=False))
        logging.info("latest-resource data refreshed.")


class Name(Mongo):
    def get_names(self, is_readable: [str, bool]) -> dict:
        if is_readable:
            aggregation = [
                {
                    "$project": {
                        "name": {
                            "$concat": [
                                "$data.info.area",
                                "$data.info.channel_cn",
                                ": ",
                                "$data.info.cnname",
                                " ",
                                "$data.info.enname",
                                " ",
                                "$data.info.aliasname",
                            ]
                        },
                        "_id": False,
                    }
                }
            ]
            query_cursor = self.db["yyets"].aggregate(aggregation)
        else:
            projection = {
                "_id": False,
                "data.info.cnname": True,
                "data.info.enname": True,
                "data.info.aliasname": True,
                "data.info.channel_cn": True,
            }
            query_cursor = self.db["yyets"].find({}, projection)

        data = []
        for i in query_cursor:
            data.extend(i.values())

        return dict(data=data)



File: yyetsweb/databases/base.py

#!/usr/bin/env python3
# coding: utf-8
import json
import logging
import os
import time

import meilisearch

from databases import db, redis_client


class Mongo:
    def __init__(self):
        self.db = db
        super().__init__()

    def is_admin(self, username: str) -> bool:
        data = self.db["users"].find_one({"username": username, "group": {"$in": ["admin"]}})
        if data:
            return True

    def is_user_blocked(self, username: str) -> str:
        r = self.db["users"].find_one({"username": username, "status.disable": True})
        if r:
            return r["status"]["reason"]

    def is_old_user(self, username: str) -> bool:
        return bool(self.db["users"].find_one({"username": username, "oldUser": True}))


class Redis:
    def __init__(self):
        self.r = redis_client
        super().__init__()

    @classmethod
    def cache(cls, timeout: int):
        def func(fun):
            def inner(*args, **kwargs):
                func_name = fun.__name__
                cache_value = cls().r.get(func_name)
                if cache_value:
                    logging.info("Retrieving %s data from redis", func_name)
                    return json.loads(cache_value)
                else:
                    logging.info("Cache expired. Executing %s", func_name)
                    res = fun(*args, **kwargs)
                    cls().r.set(func_name, json.dumps(res), ex=timeout)
                    return res

            return inner

        return func


class SearchEngine(Mongo):
    yyets_projection = {
        "data.info.cnname": 1,
        "data.info.enname": 1,
        "data.info.aliasname": 1,
        "data.info.area": 1,
        "data.info.id": 1,
        "data.info.channel_cn": 1,
        "data.info.channel": 1,
        "_id": {"$toString": "$_id"},
        "origin": "yyets",
    }

    douban_projection = {
        "_id": {"$toString": "$_id"},
        "id": "$resourceId",
        "cnname": {"$first": "$resource.data.info.cnname"},
        "enname": {"$first": "$resource.data.info.enname"},
        "aliasname": {"$first": "$resource.data.info.aliasname"},
        "area": {"$first": "$resource.data.info.area"},
        "channel_cn": {"$first": "$resource.data.info.channel_cn"},
        "channel": {"$first": "$resource.data.info.channel"},
        "origin": "yyets",
        "actors": 1,
        "directors": 1,
        "genres": 1,
        "writers": 1,
        "introduction": 1,
    }

    douban_lookup = {
        "from": "yyets",
        "localField": "resourceId",
        "foreignField": "data.info.id",
        "as": "resource",
    }
    comment_projection = {
        "username": 1,
        "date": 1,
        "comment": "$content",
        "commentID": {"$toString": "$_id"},
        "origin": "comment",
        "hasAvatar": "yes",
        "resourceID": "$resource_id",
        "resourceName": {"$first": "$resource.data.info.cnname"},
        "_id": {"$toString": "$_id"},
    }
    comment_lookup = {
        "from": "yyets",
        "localField": "resource_id",
        "foreignField": "data.info.id",
        "as": "resource",
    }

    def __init__(self):
        self.search_client = meilisearch.Client(os.getenv("MEILISEARCH"), "masterKey")
        self.yyets_index = self.search_client.index("yyets")
        self.comment_index = self.search_client.index("comment")
        self.douban_index = self.search_client.index("douban")
        super().__init__()

    def __del__(self):
        pass

    def __get_yyets(self):
        return self.db["yyets"].aggregate(
            [
                {"$project": self.yyets_projection},
                {
                    "$replaceRoot": {
                        "newRoot": {
                            "$mergeObjects": [
                                {"origin": "yyets"},
                                "$data.info",
                                {"_id": "$_id"},
                            ]
                        }
                    }
                },
            ]
        )

    def __get_comment(self):
        return self.db["comment"].aggregate(
            [
                {"$lookup": self.comment_lookup},
                {"$project": self.comment_projection},
            ]
        )

    def __get_douban(self):
        return self.db["douban"].aggregate(
            [
                {"$lookup": self.douban_lookup},
                {"$project": self.douban_projection},
            ]
        )

    def add_yyets(self):
        logging.info("Adding yyets data to search engine")
        data = list(self.__get_yyets())
        self.yyets_index.add_documents(data, primary_key="_id")

    def add_comment(self):
        logging.info("Adding comment data to search engine")
        data = list(self.__get_comment())
        self.comment_index.add_documents(data, primary_key="_id")

    def add_douban(self):
        logging.info("Adding douban data to search engine")
        data = list(self.__get_douban())
        self.douban_index.add_documents(data, primary_key="_id")

    def search_yyets(self, keyword: "str"):
        return self.yyets_index.search(keyword, {"matchingStrategy": "all"})["hits"]

    def search_comment(self, keyword: "str"):
        return self.comment_index.search(keyword, {"matchingStrategy": "all"})["hits"]

    def search_douban(self, keyword: "str"):
        return self.douban_index.search(keyword, {"matchingStrategy": "all"})["hits"]

    def run_import(self):
        t0 = time.time()
        self.add_yyets()
        self.add_comment()
        self.add_douban()
        logging.info(f"Import data to search engine in {time.time() - t0:.2f}s")

    def __monitor(self, col, fun):
        cursor = self.db[col].watch()
        for change in cursor:
            op_type = change["operationType"]
            _id = change["documentKey"]["_id"]
            search_index = getattr(self, f"{col}_index")
            logging.info("%s %s change stream for %s", col, op_type, _id)

            if op_type == "delete":
                search_index.delete_document(_id)
            else:
                data = fun(_id)
                search_index.add_documents(data, primary_key="_id")

    def monitor_yyets(self):
        def get_data(_id) -> list:
            data = self.db.yyets.find_one({"_id": _id}, projection=self.yyets_projection)["data"]["info"]
            data["_id"] = str(_id)
            data["origin"] = "yyets"
            return [data]

        self.__monitor("yyets", get_data)

    def monitor_douban(self):
        def get_data(_id) -> list:
            data = self.db.douban.aggregate(
                [
                    {"$match": {"_id": _id}},
                    {"$lookup": self.douban_lookup},
                    {"$project": self.douban_projection},
                ]
            )
            return list(data)

        self.__monitor("douban", get_data)

    def monitor_comment(self):
        def get_data(_id) -> list:
            data = self.db.comment.aggregate(
                [
                    {"$match": {"_id": _id}},
                    {"$lookup": self.comment_lookup},
                    {"$project": self.comment_projection},
                ]
            )
            return list(data)

        self.__monitor("comment", get_data)



File: yyetsweb/databases/oauth.py

#!/usr/bin/env python3
# coding: utf-8
from hashlib import sha256

from common.utils import ts_date
from databases.base import Mongo


class OAuthRegister(Mongo):
    def add_user(self, username, ip, browser, uid, source: "str"):
        uid = str(uid)
        # username = "Benny"
        user = self.db["users"].find_one({"uid": uid, "source": source})
        if user and user.get("password"):
            # 直接注册的用户
            return {"status": "fail", "message": "第三方登录失败，用户名已存在"}
        elif user:
            # 已存在的oauth用户
            return {"status": "success", "message": "欢迎回来，即将跳转首页", "username": username}
        else:
            # 第一次oauth登录，假定一定会成功
            # TODO GitHub可以改用户名的，但是uid不会变，也许需要加unique index
            self.db["users"].insert_one(
                {
                    "username": username,
                    "date": ts_date(),
                    "ip": ip,
                    "browser": browser,
                    "oldUser": True,
                    "source": source,
                    "uid": uid,
                    "hash": sha256(username.encode("u8")).hexdigest(),
                }
            )
            return {
                "status": "success",
                "message": "第三方登录成功，即将跳转首页",
                "username": username,
            }


class GitHubOAuth2Login(OAuthRegister):
    pass


class MSOAuth2Login(OAuthRegister):
    pass


class GoogleOAuth2Login(OAuthRegister):
    pass


class TwitterOAuth2Login(OAuthRegister):
    pass


class FacebookAuth2Login(OAuthRegister):
    pass



File: yyetsweb/common/sync.py

#!/usr/bin/env python3
# coding: utf-8
import contextlib
import logging
import os
import random
import re
import time
from copy import deepcopy

import requests
from bs4 import BeautifulSoup
from tqdm import tqdm

from databases.base import Mongo
from databases.douban import Douban


class BaseSync:
    def __init__(self):
        self.mongo = Mongo()
        self.yyets = self.mongo.db["yyets"]
        self.sync = self.mongo.db["sync"]
        self.structure = {
            "status": 1,
            "info": "OK",
            "data": {
                "info": {
                    "id": None,
                    "cnname": "",
                    "enname": " ",
                    "aliasname": "",
                    "channel": "",
                    "channel_cn": "",
                    "area": "日本",
                    "show_type": "",
                    "expire": "",
                    "views": 0,
                    "year": [],
                },
                "list": [
                    {
                        "season_num": "101",
                        "season_cn": "单剧",
                        "items": {"MP4": []},
                        "formats": [
                            "MP4",
                        ],
                    }
                ],
            },
        }
        self.session = requests.Session()
        self.session.headers.update(
            {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36"
            }
        )

    @staticmethod
    def sleep(times=1):
        time.sleep(random.random() * times)


class Zhuixinfan(BaseSync):
    def run(self):
        zhuixinfan = "http://www.fanxinzhui.com/rr/{}"
        start = (self.sync.find_one({"name": "zhuixinfan"}) or {}).get(
            "resource_id", os.getenv("ZHUIXINFAN_START", 20)
        )
        end = os.getenv("ZHUIXINFAN_END", 2500)
        for i in range(start, end):
            url = zhuixinfan.format(i)
            html = self.session.get(zhuixinfan.format(i)).content.decode("u8")
            self.sleep()
            if html != "资源不存在":
                self.build_data(html, url)

        self.sync.update_one({"name": "zhuixinfan"}, {"$set": {"resource_id": end}})
        logging.info("Zhuixinfan Finished")

    def build_data(self, html, link):
        structure = deepcopy(self.structure)
        if "更新至" in html or re.findall(r"全\d+回", html):
            channel, channel_cn = "tv", "日剧"
        else:
            channel, channel_cn = "movie", "日影"
        soup = BeautifulSoup(html, "html.parser")
        title = soup.find_all("div", class_="resource_title")[0].h2.text
        chn = soup.title.text.split("_")[0]
        eng = title.replace(chn, "").split("(")[0].strip()
        year = int("".join(re.findall(r"\d{4}", title)[-1]).strip())
        structure["data"]["info"]["cnname"] = chn
        structure["data"]["info"]["enname"] = eng
        structure["data"]["info"]["year"] = [year]
        structure["data"]["info"]["source"] = link
        structure["data"]["info"]["channel"] = channel
        structure["data"]["info"]["channel_cn"] = channel_cn

        logging.info("Building data for %s -  %s", chn, link)

        li = soup.find("ul", class_="item_list")
        if li:
            li = li.find_all("li")
        else:
            li = []

        for p in li:
            resource = {
                "itemid": "",
                "episode": p.span.text,
                "name": p.span.nextSibling.text,
                "size": "unknown",
                "yyets_trans": 0,
                "dateline": str(int(time.time())),
                "files": [],
            }

            res = p.find("p", class_="way")
            if res.span is None:
                continue
            links = res.find_all("a")
            for item in links:
                content = item["href"]
                if "ed2k" in content:
                    resource["files"].append(
                        {"way": "1", "way_cn": "电驴", "address": content, "passwd": ""}
                    )
                elif "magnet" in content:
                    resource["files"].append(
                        {"way": "2", "way_cn": "磁力", "address": content, "passwd": ""}
                    )
                elif "pan.baidu" in content:
                    baidu_password = res.span.a.nextSibling.nextSibling.text
                    resource["files"].append(
                        {
                            "way": "13",
                            "way_cn": "百度网盘",
                            "address": content,
                            "passwd": baidu_password,
                        }
                    )
                elif "weiyun" in content:
                    resource["files"].append(
                        {"way": "14", "way_cn": "微云", "address": content, "passwd": ""}
                    )
                else:
                    logging.debug("Unknown link: %s", content)

            structure["data"]["list"][0]["items"]["MP4"].append(resource)

        self.update_yyets(structure)

    def update_yyets(self, data):
        source = data["data"]["info"]["source"]
        exists = self.yyets.find_one({"data.info.source": source})
        already_cond = {"data.info.cnname": data["data"]["info"]["cnname"]}
        already_in = self.yyets.find_one(already_cond)
        if already_in:
            logging.info("Already in old yyets, updating data.info.source: %s", source)
            self.yyets.update_one(already_cond, {"$set": {"data.info.source": source}})
        elif exists:
            logging.info("Updating new data.info.id: %s", source)
            self.yyets.update_one(
                {"data.info.source": source},
                {"$set": {"data.list": data["data"]["list"]}},
            )
        else:
            last_id = 90000
            last = self.yyets.find_one(
                {"data.info.id": {"$gte": last_id}}, sort=[("data.info.id", -1)]
            )
            if last:
                last_id = last["data"]["info"]["id"] + 1
            logging.info("Inserting data.info.id: %s", last_id)
            data["data"]["info"]["id"] = last_id
            self.yyets.insert_one(data.copy())


class YYSub(BaseSync):
    def get_lastest_id(self):
        url = "https://www.yysub.net/resourcelist?channel=&area=&category=&year=&tvstation=&sort=pubdate&page=1"
        html = self.session.get(url).content.decode("u8")
        soup = BeautifulSoup(html, "html.parser")
        x = soup.find("div", class_="resource-showlist has-point")
        return int(x.ul.li.div.a["href"].split("/")[-1])

    def get_channel_cn(self, channel, area):
        if len(area) == 2 and channel == "tv":
            return f"{area[0]}剧"
        if channel == "movie":
            return "电影"
        return ""

    def run(self):
        logging.info("Starting to sync YYSub...")
        structure = deepcopy(self.structure)
        end = self.get_lastest_id() + 1
        # end = 41566
        start = (self.sync.find_one({"name": "yysub"}) or {}).get("resource_id", 41557)
        api = "https://m.yysub.net/api/resource/{}"
        for i in range(start, end):
            resp = self.session.get(api.format(i))
            self.sleep()
            if resp.status_code != 200:
                continue
            data = resp.json()["data"]
            if data.get("cnname"):
                logging.info("Found valid resource: %s - %s", data["cnname"], i)
                channel_cn = self.get_channel_cn(data["channel"], data["area"])
                structure["data"]["info"]["id"] = i
                structure["data"]["info"]["cnname"] = data["cnname"]
                structure["data"]["info"]["enname"] = data["enname"]
                structure["data"]["info"]["aliasname"] = data["aliasname"]
                structure["data"]["info"]["channel"] = data["channel"]
                structure["data"]["info"]["channel_cn"] = (
                    data["channel_cn"] or channel_cn
                )
                structure["data"]["info"]["area"] = data["area"]
                structure["data"]["list"] = []
                structure["data"]["info"][
                    "source"
                ] = f"https://www.yysub.net/resource/{i}"
                self.insert_data(structure.copy())

        self.sync.update_one(
            {"name": "yysub"}, {"$set": {"resource_id": end}}, upsert=True
        )
        logging.info("YYsub Finished")

    def insert_data(self, data):
        rid = data["data"]["info"]["id"]
        self.yyets.update_one({"data.info.id": rid}, {"$set": data}, upsert=True)


def sync_douban():
    douban = Douban()
    session = requests.Session()
    ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4280.88 Safari/537.36"
    session.headers.update({"User-Agent": ua})

    yyets_data = douban.db["yyets"].aggregate(
        [
            {"$group": {"_id": None, "ids": {"$push": "$data.info.id"}}},
            {"$project": {"_id": 0, "ids": 1}},
        ]
    )
    douban_data = douban.db["douban"].aggregate(
        [
            {"$group": {"_id": None, "ids": {"$push": "$resourceId"}}},
            {"$project": {"_id": 0, "ids": 1}},
        ]
    )

    id1 = next(yyets_data)["ids"]
    id2 = next(douban_data)["ids"]
    rids = list(set(id1).difference(id2))
    rids.remove(233)
    logging.info("resource id complete %d", len(rids))
    for rid in tqdm(rids):
        with contextlib.suppress(Exception):
            d = douban.find_douban(rid)
            logging.info("Processed %s, length %d", rid, len(d))

    logging.info("ALL FINISH!")


if __name__ == "__main__":
    a = Zhuixinfan()
    # a.build_data(open("1.html").read(), "https://www.zhuixinfan.com/resource/1.html")
    a.run()
    # b = YYSub()
    # b.run()



File: yyetsweb/common/dump_db.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - dump_db.py
# 2/4/22 18:10
#

__author__ = "Benny <benny.think@gmail.com>"

import contextlib
import json
import logging
import os
import pathlib
import sqlite3
import subprocess
import time
import zipfile

import pymongo
import pymysql
import pymysql.cursors
from tqdm import tqdm

from common.utils import setup_logger

setup_logger()
data_path = pathlib.Path(__file__).parent.parent.joinpath("templates", "dump")
data_path.mkdir(exist_ok=True)
sqlite_file = data_path.joinpath("yyets.db")

CHUNK_SIZE = 1000


def SQLite():
    return sqlite3.connect(sqlite_file, check_same_thread=False)


def MySQL():
    return pymysql.connect(host="mysql", user="root", passwd="root", charset="utf8mb4")


def MongoDB():
    return pymongo.MongoClient("mongo", 27017, connect=False)


def read_resource():
    logging.info("Reading resource from mongo")
    client = MongoDB()
    data = client["zimuzu"]["yyets"].aggregate(
        [
            {
                "$project": {
                    "data.info.id": 1,
                    "data.info.cnname": 1,
                    "data.info.enname": 1,
                    "data.info.aliasname": 1,
                    "data.info.views": 1,
                    "data.info.area": 1,
                    "fullDocument": "$data",
                }
            },
            {
                "$replaceRoot": {
                    "newRoot": {
                        "$mergeObjects": [
                            "$data.info",
                            {"data": "$fullDocument"},
                        ]
                    }
                }
            },
        ]
    )
    return data


def read_comment():
    logging.info("Reading comment from mongo")
    client = MongoDB()
    res = client["zimuzu"]["comment"].aggregate(
        [
            {
                "$project": {
                    "_id": 0,
                    "content": 1,
                    "date": 1,
                    "resource_id": 1,
                    "browser": "browser",
                    "username": "username",
                }
            }
        ]
    )
    return res


def prepare_mysql():
    logging.info("Preparing mysql")
    db_sql = "create database zimuzu;"
    resource_sql = """
        create table yyets
        (
            resource_id int          null,
            cnname      varchar(256) null,
            enname     varchar(256) null,
            aliasname   varchar(256) null,
            area varchar(32),
            views int null,
            data        longtext     null,
            douban longtext null,
            image blob null
        ) charset utf8mb4;
        """
    comment_sql = """
        create table comment
        (
            date     varchar(256) null,
            content  longtext null,
            resource_id     varchar(256) null,
            browser     varchar(256) null,
            username     varchar(256) null
        ) charset utf8mb4;
        """
    con = MySQL()
    cur = con.cursor()
    cur.execute(db_sql)
    cur.execute("use zimuzu")
    cur.execute(resource_sql)
    cur.execute(comment_sql)
    con.commit()
    con.close()


def prepare_sqlite():
    logging.info("Preparing sqlite")
    con = SQLite()
    cur = con.cursor()
    resource_sql = """
            create table yyets
            (
                resource_id int          null,
                cnname      varchar(256) null,
                enname     varchar(256) null,
                aliasname   varchar(256) null,
                area varchar(32),
                views int null,
                data        longtext     null,
                douban longtext null,
                image blob null
            );
            """
    comment_sql = """
            create table comment
            (
             date     varchar(256) null,
                content  longtext null,
               
                resource_id     varchar(256) null,
                browser     varchar(256) null,
                username     varchar(256) null
            );
            """

    cur.execute(resource_sql)
    cur.execute(comment_sql)
    con.commit()
    con.close()


def dump_resource():
    res = read_resource()
    # insert into mysql
    batch_data = []
    mb = []
    client = MongoDB()
    db = client["zimuzu"]
    for each in tqdm(res, total=db["yyets"].count_documents({})):
        line = list(each.values())
        line[-1] = json.dumps(line[-1], ensure_ascii=False)
        line.extend(["", ""])
        batch_data.append(line)
        mb.append(each)
        if len(batch_data) == CHUNK_SIZE:
            sql1 = "insert into yyets values (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
            sql2 = "insert into yyets values (?, ?, ?, ?, ?,?,?,?,?)"
            insert_func(batch_data, mb, sql1, sql2, "yyets")
            batch_data = []
            mb = []


def insert_func(batch_data, mb, sql1, sql2, col_name=None):
    mysql_con = MySQL()
    sqlite_con = SQLite()
    mysql_cur = mysql_con.cursor()
    sqlite_cur = sqlite_con.cursor()
    client = MongoDB()
    col = client["share"][col_name]
    mysql_cur.execute("use zimuzu")

    mysql_cur.executemany(sql1, batch_data)
    sqlite_cur.executemany(sql2, batch_data)
    col.insert_many(mb)
    mysql_con.commit()
    sqlite_con.commit()
    mysql_con.close()
    sqlite_con.close()


def dump_comment():
    res = read_comment()
    batch_data = []
    mb = []
    client = MongoDB()
    for each in tqdm(res, total=client["zimuzu"]["comment"].count_documents({})):
        batch_data.append(list(each.values()))
        mb.append(each)
        if len(batch_data) == CHUNK_SIZE:
            sql1 = "insert into comment values (%s, %s, %s, %s,%s)"
            sql2 = "insert into comment values ( ?, ?, ?, ?,?)"
            insert_func(batch_data, mb, sql1, sql2, "comment")
            batch_data = []
            mb = []


def zip_file():
    logging.info("Zipping SQLite...")
    p = data_path.joinpath("yyets_sqlite.zip")
    with zipfile.ZipFile(p, "w", zipfile.ZIP_DEFLATED) as zf:
        zf.write(sqlite_file, "yyets_sqlite.db")

    logging.info("Dumping MySQL...")
    subprocess.check_output("mysqldump -h mysql -u root -proot zimuzu > zimuzu.sql", shell=True)
    p = data_path.joinpath("yyets_mysql.zip")
    logging.info("Zipping MySQL...")
    with zipfile.ZipFile(p, "w", zipfile.ZIP_DEFLATED) as zf:
        zf.write("zimuzu.sql")

    logging.info("Dumping MongoDB")
    subprocess.check_output(
        "mongodump -h mongo -d share --gzip --archive=" + data_path.joinpath("yyets_mongo.gz").as_posix(),
        shell=True,
    )


def cleanup():
    # this function will never occur any errors!
    logging.info("Cleaning up...")
    with contextlib.suppress(Exception):
        con = MySQL()
        con.cursor().execute("drop database zimuzu")
        con.close()
    with contextlib.suppress(Exception):
        os.unlink(sqlite_file)
    with contextlib.suppress(Exception):
        MongoDB().drop_database("share")
    with contextlib.suppress(Exception):
        os.unlink("zimuzu.sql")


def entry_dump():
    cleanup()
    t0 = time.time()
    prepare_mysql()
    prepare_sqlite()
    dump_resource()
    dump_comment()
    logging.info("Write done! Time used: %.2fs" % (time.time() - t0))
    zip_file()
    cleanup()
    logging.info("Total time used: %.2fs" % (time.time() - t0))


if __name__ == "__main__":
    t0 = time.time()
    entry_dump()
    logging.info("Total time used: %.2fs" % (time.time() - t0))



File: yyetsweb/common/__init__.py

#!/usr/bin/env python3
# coding: utf-8

# YYeTsBot - __init__.py
# 2023-03-17  18:57

from common.utils import setup_logger

setup_logger()



File: yyetsweb/common/utils.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - utils.py
# 6/16/21 21:42
#

__author__ = "Benny <benny.think@gmail.com>"

import contextlib
import json
import logging
import os
import pathlib
import re
import smtplib
import time
from datetime import datetime
from email.header import Header
from email.mime.text import MIMEText
from email.utils import formataddr, parseaddr
from hashlib import sha256

import coloredlogs
import pytz
import requests
from akismet import Akismet
from jinja2 import Template

from databases.base import Redis


def setup_logger():
    coloredlogs.install(
        level=logging.INFO,
        fmt="[%(asctime)s %(filename)s:%(lineno)d %(levelname).1s] %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )


def hide_phone(data: list):
    for item in data:
        if item["username"].isdigit() and len(item["username"]) == 11:
            item["hash"] = sha256(item["username"].encode("u8")).hexdigest()
            item["username"] = mask_phone(item["username"])
    return data


def mask_phone(num):
    return re.sub(r"(\d{3})\d{4}(\d{4})", r"\g<1>****\g<2>", num)


def ts_date(ts=None):
    # Let's always set the timezone to CST
    timestamp = ts or time.time()
    return datetime.fromtimestamp(timestamp, pytz.timezone("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S")


def _format_addr(s):
    name, addr = parseaddr(s)
    return formataddr((Header(name, "utf-8").encode(), addr))


def generate_body(context):
    template = pathlib.Path(__file__).parent.parent.joinpath("templates", "email_template.html")
    with open(template) as f:
        return Template(f.read()).render(**context)


def send_mail(to: str, subject: str, context: dict):
    user = os.getenv("email_user")
    password = os.getenv("email_password")
    host = os.getenv("email_host", "localhost")
    port = os.getenv("email_port", "1025")  # mailhog
    from_addr = os.getenv("from_addr", "hello@yyets.click")

    msg = MIMEText(generate_body(context), "html", "utf-8")
    msg["From"] = _format_addr("YYeTs <%s>" % from_addr)
    msg["To"] = _format_addr(to)
    msg["Subject"] = Header(subject, "utf-8").encode()

    logging.info("logging to mail server...")
    if port == "1025":
        server = smtplib.SMTP(host, int(port))
    else:
        server = smtplib.SMTP_SSL(host, int(port))
    server.login(user, password)
    logging.info("sending email to %s", to)
    server.sendmail(from_addr, [to], msg.as_string())
    server.quit()


def check_spam(ip, ua, author, content) -> int:
    # 0 means okay
    token = os.getenv("askismet")
    whitelist: "list" = os.getenv("whitelist", "").split(",")
    if author in whitelist:
        return 0
    if token:
        with contextlib.suppress(Exception):
            akismet = Akismet(token, blog="https://yyets.click/")

            return akismet.check(
                ip,
                ua,
                comment_author=author,
                blog_lang="zh_cn",
                comment_type="comment",
                comment_content=content,
            )
    return 0


class Cloudflare(Redis):
    key = "cf-blacklist-ip"
    expire = "cf-expire"

    def __init__(self):
        self.account_id = "e8d3ba82fe9e9a41cceb0047c2a2ab4f"
        self.item_id = "3740654e0b104053b3e5d0a71fe87b33"
        self.endpoint = "https://api.cloudflare.com/client/v4/accounts/{}/rules/lists/{}/items".format(
            self.account_id, self.item_id
        )
        self.session = requests.Session()
        self.session.headers.update({"Authorization": "Bearer %s" % os.getenv("CF_TOKEN")})
        super().__init__()

    def get_old_ips(self) -> dict:
        cache = self.r.get(self.key)
        if cache:
            cache = json.loads(cache)
            logging.info("Cache found with %s IPs", len(cache))
            if len(cache) > 10000:
                return cache[:5000]
            return cache
        else:
            data = self.session.get(self.endpoint).json()
            result = data.get("result", [])
            cursor = data.get("result_info", {}).get("cursors", {}).get("after")
            while cursor:
                logging.info("Fetching next page with cursor %s", cursor)
                data = self.session.get(self.endpoint, params={"cursor": cursor}).json()
                result.extend(data["result"])
                cursor = data.get("result_info", {}).get("cursors", {}).get("after")
            logging.info("Got %s IPs", len(result))
            return result

    def ban_new_ip(self, ip):
        if ":" in ip:
            ip = ip.rsplit(":", 4)[0] + "::/64"
        old_ips = [d["ip"] for d in self.get_old_ips()]
        old_ips.append(ip)
        body = [{"ip": i} for i in set(old_ips)]
        self.r.set(self.key, json.dumps(body))
        if not self.r.exists(self.expire):
            resp = self.session.put(self.endpoint, json=body)
            logging.info(resp.json())
            self.r.set(self.expire, 1, ex=120)

    def clear_fw(self):
        logging.info("Clearing firewall rules")
        self.session.put(self.endpoint, json=[{"ip": "192.168.3.1"}])
        logging.info("Clearing cache from redis")
        self.r.delete(self.key)


if __name__ == "__main__":
    cf = Cloudflare()
    cf.ban_new_ip("192.168.1.1")



File: yyetsweb/commands/share_excel.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - share_excel.py
# 12/18/21 19:21
#

__author__ = "Benny <benny.think@gmail.com>"

import pathlib
import sys

import openpyxl

web_path = pathlib.Path(__file__).parent.parent.resolve().as_posix()
sys.path.append(web_path)
from tqdm import tqdm

from common.utils import ts_date
from databases.base import Mongo

wb = openpyxl.open("aliyun.xlsx")

data = {}

for ws in wb.worksheets:
    line = 0
    for line in range(1, ws.max_row + 1):
        name = ws.cell(line, 1).value
        link = ws.cell(line, 2).value
        line += 1
        data[name] = link

template = {
    "username": "Benny",
    "ip": "127.0.0.1",
    "date": "",
    "browser": "cli",
    "content": "",
    "resource_id": 234,
    "type": "parent",
}
col = Mongo().db["comment"]
share_doc = {
    "status": 1.0,
    "info": "OK",
    "data": {
        "info": {
            "id": 234,
            "cnname": "网友分享",
            "enname": "",
            "aliasname": "",
            "channel": "share",
            "channel_cn": "",
            "area": "",
            "show_type": "",
            "expire": "1610401225",
            "views": 0,
        },
        "list": [],
    },
}

Mongo().db["yyets"].update_one({"data.info.id": 234}, {"$set": share_doc}, upsert=True)

for name, link in tqdm(data.items()):
    template["content"] = f"{name}\n{link}"
    template["date"] = ts_date()
    col.insert_one(template.copy())



File: yyetsweb/commands/grafana_test_data.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - grafana_test_data.py
# 3/14/21 18:25
#

__author__ = "Benny <benny.think@gmail.com>"

import random
from datetime import date, timedelta

from common import Mongo

col = Mongo().client["zimuzu"]["metrics"]


def generate_date_series(start: str, end: str) -> list:
    start_int = [int(i) for i in start.split("-")]
    end_int = [int(i) for i in end.split("-")]
    sdate = date(*start_int)  # start date
    edate = date(*end_int)  # end date

    delta = edate - sdate  # as timedelta
    days = []
    for i in range(delta.days + 1):
        day = sdate + timedelta(days=i)
        days.append(day.strftime("%Y-%m-%d"))
    return days


date_series = generate_date_series("2021-02-01", "2021-03-14")

inserted = []
for date in date_series:
    inserted.append(
        {
            "date": date,
            "access": random.randint(1, 50),
            "search": random.randint(1, 50),
            "resource": random.randint(1, 50),
        }
    )

col.insert_many(inserted)



File: yyetsweb/commands/douban_fix.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - douban_fix.py
# 7/11/21 09:37
#

__author__ = "Benny <benny.think@gmail.com>"

import argparse
import pathlib
import sys

import requests

lib_path = pathlib.Path(__file__).parent.parent.resolve().as_posix()
sys.path.append(lib_path)

from databases.douban import Douban

parser = argparse.ArgumentParser(description="豆瓣数据修复")
parser.add_argument("resource_id", metavar="r", type=int, help="resource id")
parser.add_argument("douban_id", metavar="d", type=int, help="douban id")
args = parser.parse_args()
resource_id = args.resource_id
douban_id = args.douban_id

douban = Douban()
session = requests.Session()
ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
session.headers.update({"User-Agent": ua})

yyets_data = douban.db["yyets"].find_one({"data.info.id": resource_id})
search_html = ""
cname = yyets_data["data"]["info"]["cnname"]

final_data = douban.get_craw_data(cname, douban_id, resource_id, search_html, session)
douban.db["douban"].find_one_and_replace({"resourceId": resource_id}, final_data)
print("fix complete")
sys.exit(0)



File: yyetsweb/commands/common.py

#!/usr/local/bin/python3
# coding: utf-8

# YYeTsBot - common.py
# 3/26/22 10:40
#

__author__ = "Benny <benny.think@gmail.com>"

import os

import pymongo


class Mongo:
    def __init__(self):
        self.client = pymongo.MongoClient(
            host=os.getenv("MONGO", "localhost"),
            connect=False,
            connectTimeoutMS=5000,
            serverSelectionTimeoutMS=5000,
        )
        self.db = self.client["zimuzu"]

    def __del__(self):
        self.client.close()



File: yyetsweb/handlers/user.py

#!/usr/bin/env python3
# coding: utf-8
from http import HTTPStatus
from pathlib import Path

from tornado import gen, web
from tornado.concurrent import run_on_executor

from handlers.base import BaseHandler

filename = Path(__file__).name.split(".")[0]


class UserHandler(BaseHandler):
    filename = filename

    def set_login(self, username):
        self.set_secure_cookie("username", username, 365)

    @run_on_executor()
    def login(self):
        data = self.json
        username = data["username"]
        password = data["password"]
        captcha = data.get("captcha")
        captcha_id = data.get("captcha_id", "")
        ip = self.get_real_ip()
        browser = self.request.headers["user-agent"]

        response = self.instance.login_user(username, password, captcha, captcha_id, ip, browser)
        if response["status_code"] in (HTTPStatus.CREATED, HTTPStatus.OK):
            self.set_login(username)
        else:
            self.set_status(response["status_code"])

        return response

    @run_on_executor()
    def update_info(self):
        result = self.instance.update_user_info(self.current_user, self.json)
        self.set_status(result.get("status_code", HTTPStatus.IM_A_TEAPOT))
        return result

    @run_on_executor()
    def get_user_info(self) -> dict:
        username = self.get_current_user()
        if username:
            data = self.instance.get_user_info(username)
        else:
            # self.set_status(HTTPStatus.UNAUTHORIZED)
            self.clear_cookie("username")
            data = {"message": "Please try to login"}
        return data

    @gen.coroutine
    def post(self):
        resp = yield self.login()
        self.write(resp)

    @gen.coroutine
    def get(self):
        resp = yield self.get_user_info()
        self.write(resp)

        # everytime we receive a GET request to this api, we'll update last_date and last_ip
        username = self.get_current_user()
        if username:
            now_ip = self.get_real_ip()
            self.instance.update_user_last(username, now_ip)

    @gen.coroutine
    @web.authenticated
    def patch(self):
        resp = yield self.update_info()
        self.write(resp)


class UserAvatarHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def update_avatar(self):
        username = self.get_current_user()
        if not username:
            self.set_status(HTTPStatus.UNAUTHORIZED)
            self.clear_cookie("username")
            return {"message": "Please try to login"}

        file = self.request.files["image"][0]["body"]
        if len(file) > 10 * 1024 * 1024:
            self.set_status(HTTPStatus.REQUEST_ENTITY_TOO_LARGE)
            return {"message": "图片大小不可以超过10MB"}
        return self.instance.add_avatar(username, file)

    @run_on_executor()
    def get_avatar(self, username):
        user_hash = self.get_query_argument("hash", None)
        data = self.instance.get_avatar(username, user_hash)
        if data["image"]:
            self.set_header("Content-Type", data["content_type"])
            return data["image"]
        self.set_status(HTTPStatus.NOT_FOUND)
        return b""

    @gen.coroutine
    def post(self, _):
        resp = yield self.update_avatar()
        self.write(resp)

    @gen.coroutine
    def get(self, username):
        resp = yield self.get_avatar(username)
        self.write(resp)

    @gen.coroutine
    def head(self, username):
        resp = yield self.get_avatar(username)
        self.write(resp)


class LikeHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def like_data(self):
        username = self.get_current_user()
        return {"LIKE": self.instance.get_user_like(username)}

    @gen.coroutine
    @web.authenticated
    def get(self):
        resp = yield self.like_data()
        self.write(resp)

    @run_on_executor()
    def add_remove_fav(self):
        data = self.json
        resource_id = int(data["resource_id"])
        username = self.get_current_user()
        if username:
            response = self.instance.add_remove_fav(resource_id, username)
            self.set_status(response["status_code"])
        else:
            response = {"message": "请先登录"}
            self.set_status(HTTPStatus.UNAUTHORIZED)

        return response["message"]

    @gen.coroutine
    @web.authenticated
    def patch(self):
        resp = yield self.add_remove_fav()
        self.write(resp)


class UserEmailHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def verify_email(self):
        result = self.instance.verify_email(self.get_current_user(), self.json["code"])
        self.set_status(result.get("status_code"))
        return result

    @gen.coroutine
    @web.authenticated
    def post(self):
        resp = yield self.verify_email()
        self.write(resp)



File: yyetsweb/handlers/comment.py

#!/usr/bin/env python3
# coding: utf-8
from http import HTTPStatus
from pathlib import Path

from tornado import gen, web
from tornado.concurrent import run_on_executor

from common.utils import hide_phone
from handlers.base import BaseHandler

filename = Path(__file__).name.split(".")[0]


class CommentHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def get_comment(self):
        query_id = self.get_argument("resource_id", "0")
        resource_id = int(query_id) if query_id.isdigit() else 0

        size = int(self.get_argument("size", "5"))
        page = int(self.get_argument("page", "1"))
        inner_size = int(self.get_argument("inner_size", "3"))
        inner_page = int(self.get_argument("inner_page", "1"))
        comment_id = self.get_argument("comment_id", None)
        sort = self.get_argument("sort", "newest")

        if not resource_id:
            self.set_status(HTTPStatus.BAD_REQUEST)
            return {"status": False, "message": "请提供resource id"}
        comment_data = self.instance.get_comment(
            resource_id,
            page,
            size,
            sort=sort,
            inner_size=inner_size,
            inner_page=inner_page,
            comment_id=comment_id,
        )
        hide_phone((comment_data["data"]))
        return comment_data

    @run_on_executor()
    def add_comment(self):
        payload = self.json
        captcha = payload["captcha"]
        captcha_id = payload["id"]
        content = payload["content"]
        resource_id = payload["resource_id"]
        comment_id = payload.get("comment_id")

        real_ip = self.get_real_ip()
        username = self.get_current_user()
        browser = self.request.headers["user-agent"]

        result = self.instance.add_comment(
            captcha,
            captcha_id,
            content,
            resource_id,
            real_ip,
            username,
            browser,
            comment_id,
        )
        self.set_status(result["status_code"])
        return result

    @run_on_executor()
    def delete_comment(self):
        # need resource_id & id
        # payload = {"id":  "obj_id"}
        payload = self.json
        username = self.get_current_user()
        comment_id = payload["comment_id"]

        if self.instance.is_admin(username):
            result = self.instance.delete_comment(comment_id)
            self.set_status(result["status_code"])
            return result
        else:
            self.set_status(HTTPStatus.UNAUTHORIZED)
            return {"count": 0, "message": "You're unauthorized to delete comment."}

    @gen.coroutine
    def get(self):
        resp = yield self.get_comment()
        self.write(resp)

    @gen.coroutine
    @web.authenticated
    def post(self):
        resp = yield self.add_comment()
        self.write(resp)

    @gen.coroutine
    @web.authenticated
    def delete(self):
        resp = yield self.delete_comment()
        self.write(resp)


class CommentReactionHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def comment_reaction(self):
        self.json.update(method=self.request.method)
        username = self.get_current_user()
        result = self.instance.react_comment(username, self.json)
        self.set_status(result.get("status_code"))
        return result

    @gen.coroutine
    @web.authenticated
    def post(self):
        resp = yield self.comment_reaction()
        self.write(resp)

    @gen.coroutine
    @web.authenticated
    def delete(self):
        resp = yield self.comment_reaction()
        self.write(resp)


class CommentChildHandler(CommentHandler):
    filename = filename

    @run_on_executor()
    def get_comment(self):
        parent_id = self.get_argument("parent_id", "0")
        size = int(self.get_argument("size", "3"))
        page = int(self.get_argument("page", "1"))

        if not parent_id:
            self.set_status(HTTPStatus.BAD_REQUEST)
            return {"status": False, "message": "请提供 parent_id"}
        comment_data = self.instance.get_comment(parent_id, page, size)
        hide_phone((comment_data["data"]))
        return comment_data

    @gen.coroutine
    def get(self):
        resp = yield self.get_comment()
        self.write(resp)


class CommentNewestHandler(CommentHandler):
    filename = filename

    @run_on_executor()
    def get_comment(self):
        size = int(self.get_argument("size", "5"))
        page = int(self.get_argument("page", "1"))

        comment_data = self.instance.get_comment(page, size)
        hide_phone((comment_data["data"]))
        return comment_data

    @gen.coroutine
    def get(self):
        resp = yield self.get_comment()
        self.write(resp)


class CommentSearchHandler(CommentHandler):
    filename = filename

    @run_on_executor()
    def search_comment(self):
        size = int(self.get_argument("size", "5"))
        page = int(self.get_argument("page", "1"))
        keyword = self.get_argument("keyword", "")
        comment_data = self.instance.get_comment(page, size, keyword)
        hide_phone((comment_data["data"]))
        return comment_data

    @gen.coroutine
    def get(self):
        resp = yield self.search_comment()
        self.write(resp)


class NotificationHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def get_notification(self):
        username = self.get_current_user()
        size = int(self.get_argument("size", "5"))
        page = int(self.get_argument("page", "1"))

        return self.instance.get_notification(username, page, size)

    @run_on_executor()
    def update_notification(self):
        username = self.get_current_user()
        verb = self.json["verb"]
        comment_id = self.json["comment_id"]
        if verb not in ["read", "unread"]:
            self.set_status(HTTPStatus.BAD_REQUEST)
            return {"status": False, "message": "verb: read or unread"}
        self.set_status(HTTPStatus.CREATED)
        return self.instance.update_notification(username, verb, comment_id)

    @gen.coroutine
    @web.authenticated
    def get(self):
        resp = yield self.get_notification()
        self.write(resp)

    @gen.coroutine
    @web.authenticated
    def patch(self):
        resp = yield self.update_notification()
        self.write(resp)



File: yyetsweb/handlers/douban.py

#!/usr/bin/env python3
# coding: utf-8
from http import HTTPStatus
from pathlib import Path

import filetype
from tornado import gen
from tornado.concurrent import run_on_executor

from handlers.base import BaseHandler

filename = Path(__file__).name.split(".")[0]


class DoubanHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def douban_data(self):
        rid = self.get_query_argument("resource_id")
        data = self.instance.get_douban_data(int(rid))
        data.pop("posterData", None)
        if not data:
            self.set_status(HTTPStatus.NOT_FOUND)
        return data

    def get_image(self) -> bytes:
        rid = self.get_query_argument("resource_id")
        return self.instance.get_douban_image(int(rid))

    @gen.coroutine
    def get(self):
        _type = self.get_query_argument("type", None)
        if _type == "image":
            data = self.get_image()
            self.set_header("content-type", filetype.guess_mime(data))
            self.write(data)
        else:
            resp = yield self.douban_data()
            self.write(resp)


class DoubanReportHandler(BaseHandler):
    class_name = "DoubanReportResource"

    @run_on_executor()
    def get_error(self):
        return self.instance.get_error()

    @run_on_executor()
    def report_error(self):
        data = self.json
        user_captcha = data["captcha_id"]
        captcha_id = data["id"]
        content = data["content"]
        resource_id = data["resource_id"]
        returned = self.instance.report_error(user_captcha, captcha_id, content, resource_id)
        status_code = returned.get("status_code", HTTPStatus.CREATED)
        self.set_status(status_code)
        return self.instance.report_error(user_captcha, captcha_id, content, resource_id)

    @gen.coroutine
    def post(self):
        resp = yield self.report_error()
        self.write(resp)

    @gen.coroutine
    def get(self):
        resp = yield self.get_error()
        self.write(resp)



File: yyetsweb/handlers/__init__.py

#!/usr/bin/env python3
# coding: utf-8
import json

from tornado import escape

from common.utils import Cloudflare, setup_logger

setup_logger()
cf = Cloudflare()
escape.json_encode = lambda value: json.dumps(value, ensure_ascii=False)



File: yyetsweb/handlers/other.py

#!/usr/bin/env python3
# coding: utf-8
import logging
import os
import pathlib
import time
from hashlib import sha1
from http import HTTPStatus
from pathlib import Path

from tornado import gen, web
from tornado.concurrent import run_on_executor

from common.utils import ts_date
from databases.base import Redis
from handlers.base import BaseHandler

filename = Path(__file__).name.split(".")[0]


class AnnouncementHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def get_announcement(self):
        size = int(self.get_argument("size", "5"))
        page = int(self.get_argument("page", "1"))
        return self.instance.get_announcement(page, size)

    @run_on_executor()
    def add_announcement(self):
        username = self.get_current_user()
        if not self.instance.is_admin(username):
            self.set_status(HTTPStatus.FORBIDDEN)
            return {"message": "只有管理员可以设置公告"}

        payload = self.json
        content = payload["content"]
        real_ip = self.get_real_ip()
        browser = self.request.headers["user-agent"]

        self.instance.add_announcement(username, content, real_ip, browser)
        self.set_status(HTTPStatus.CREATED)
        return {"message": "添加成功"}

    @gen.coroutine
    def get(self):
        resp = yield self.get_announcement()
        self.write(resp)

    @gen.coroutine
    @web.authenticated
    def post(self):
        resp = yield self.add_announcement()
        self.write(resp)


class DBDumpHandler(BaseHandler):
    @staticmethod
    def sizeof_fmt(num: int, suffix="B"):
        for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
            if abs(num) < 1024.0:
                return "%3.1f%s%s" % (num, unit, suffix)
            num /= 1024.0
        return "%.1f%s%s" % (num, "Yi", suffix)

    @staticmethod
    def checksum(file_path) -> str:
        sha = sha1()
        try:
            with open(file_path, "rb") as f:
                sha.update(f.read())
                checksum = sha.hexdigest()
        except Exception as e:
            checksum = str(e)

        return checksum

    @run_on_executor()
    @Redis.cache(3600)
    def get_hash(self):
        file_list = [
            "templates/dump/yyets_mongo.gz",
            "templates/dump/yyets_mysql.zip",
            "templates/dump/yyets_sqlite.zip",
        ]
        result = {}
        for fp in file_list:
            checksum = self.checksum(fp)
            creation = ts_date(os.stat(fp).st_ctime)
            size = self.sizeof_fmt(os.stat(fp).st_size)
            result[Path(fp).name] = {
                "checksum": checksum,
                "date": creation,
                "size": size,
            }

        return result

    @gen.coroutine
    def get(self):
        resp = yield self.get_hash()
        self.write(resp)


class CategoryHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def get_data(self):
        self.json = {k: self.get_argument(k) for k in self.request.arguments}
        self.json["size"] = int(self.json.get("size", 15))
        self.json["page"] = int(self.json.get("page", 1))
        self.json["douban"] = self.json.get("douban", False)
        return self.instance.get_category(self.json)

    @gen.coroutine
    def get(self):
        resp = yield self.get_data()
        self.write(resp)


class CaptchaHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def verify_captcha(self):
        data = self.json
        captcha_id = data.get("id", None)
        userinput = data.get("captcha", None)
        if captcha_id is None or userinput is None:
            self.set_status(HTTPStatus.BAD_REQUEST)
            return "Please supply id or captcha parameter."
        returned = self.instance.verify_code(userinput, captcha_id)
        status_code = returned.get("status")
        if not status_code:
            self.set_status(HTTPStatus.FORBIDDEN)
        return returned

    @run_on_executor()
    def captcha(self):
        request_id = self.get_argument("id", None)
        if request_id is None:
            self.set_status(HTTPStatus.BAD_REQUEST)
            return "Please supply id parameter."

        return self.instance.get_captcha(request_id)

    @gen.coroutine
    def get(self):
        resp = yield self.captcha()
        self.write(resp)

    @gen.coroutine
    def post(self):
        resp = yield self.verify_captcha()
        self.write(resp)


class BlacklistHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def get_black_list(self):
        return self.instance.get_black_list()

    @gen.coroutine
    def get(self):
        resp = yield self.get_black_list()
        self.write(resp)


class SpamProcessHandler(BaseHandler):
    filename = filename

    def process(self, method):
        obj_id = self.json.get("obj_id")
        token = self.json.get("token")
        ua = self.request.headers["user-agent"]
        ip = self.get_real_ip()
        logging.info("Authentication %s(%s) for spam API now...", ua, ip)
        if token == os.getenv("TOKEN"):
            return getattr(self.instance, method)(obj_id)
        else:
            self.set_status(HTTPStatus.FORBIDDEN)
            return {
                "status": False,
                "message": "This token is not allowed to access this API",
            }

    @gen.coroutine
    def post(self):
        self.write(self.process("restore_spam"))

    @gen.coroutine
    def delete(self):
        self.write(self.process("ban_spam"))



File: yyetsweb/handlers/grafana.py

#!/usr/bin/env python3
# coding: utf-8
import json
import time
from datetime import date, timedelta
from http import HTTPStatus
from pathlib import Path

from tornado import gen
from tornado.concurrent import run_on_executor

from handlers.base import BaseHandler

filename = Path(__file__).name.split(".")[0]


class MetricsHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def set_metrics(self):
        payload = self.json
        metrics_type = payload.get("type", self.get_query_argument("type", "unknown"))

        self.instance.set_metrics(metrics_type)
        self.set_status(HTTPStatus.CREATED)
        return {}

    @run_on_executor()
    def get_metrics(self):
        if not self.instance.is_admin(self.get_current_user()):
            self.set_status(HTTPStatus.NOT_FOUND)
            return ""

        # only return latest 7 days. with days parameter to generate different range
        from_date = self.get_query_argument("from", None)
        to_date = self.get_query_argument("to", None)
        if to_date is None:
            to_date = time.strftime("%Y-%m-%d", time.localtime())
        if from_date is None:
            from_date = time.strftime("%Y-%m-%d", time.localtime(time.time() - 3600 * 24 * 7))

        return self.instance.get_metrics(from_date, to_date)

    @gen.coroutine
    def get(self):
        resp = yield self.get_metrics()
        self.write(resp)

    @gen.coroutine
    def post(self):
        resp = yield self.set_metrics()
        self.write(resp)

    @gen.coroutine
    def options(self):
        self.add_tauri()
        self.set_header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
        self.set_status(HTTPStatus.NO_CONTENT)
        self.finish()


class GrafanaIndexHandler(BaseHandler):
    filename = filename

    def get(self):
        self.write({})


class GrafanaSearchHandler(BaseHandler):
    filename = filename

    def post(self):
        data = [
            "resource",
            "top",
            "home",
            "search",
            "extra",
            "discuss",
            "multiDownload",
            "download",
            "user",
            "share",
            "me",
            "database",
            "help",
            "backOld",
            "favorite",
            "unFavorite",
            "comment",
        ]
        self.write(json.dumps(data))


class GrafanaQueryHandler(BaseHandler):
    filename = filename

    @staticmethod
    def generate_date_series(start: str, end: str) -> list:
        start_int = [int(i) for i in start.split("-")]
        end_int = [int(i) for i in end.split("-")]
        sdate = date(*start_int)  # start date
        edate = date(*end_int)  # end date

        delta = edate - sdate  # as timedelta
        days = []
        for i in range(delta.days + 1):
            day = sdate + timedelta(days=i)
            days.append(day.strftime("%Y-%m-%d"))
        return days

    @staticmethod
    def time_str_int(text):
        return time.mktime(time.strptime(text, "%Y-%m-%d"))

    def post(self):
        payload = self.json
        start = payload["range"]["from"].split("T")[0]
        end = payload["range"]["to"].split("T")[0]
        date_series = self.generate_date_series(start, end)
        targets = [i["target"] for i in payload["targets"] if i["target"]]
        grafana_data = []
        for target in targets:
            data_points = []
            result = self.instance.get_grafana_data(date_series)
            i: dict
            for i in result:
                datum = [i[target], self.time_str_int(i["date"]) * 1000] if i.get(target) else []
                data_points.append(datum)
            temp = {"target": target, "datapoints": data_points}
            grafana_data.append(temp)
        self.write(json.dumps(grafana_data))



File: yyetsweb/handlers/resources.py

#!/usr/bin/env python3
# coding: utf-8
import os
import platform
from http import HTTPStatus
from pathlib import Path

from tornado import gen
from tornado.concurrent import run_on_executor

from handlers import cf
from handlers.base import BaseHandler

filename = Path(__file__).name.split(".")[0]


class ResourceHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def get_resource_data(self):
        query = self.get_query_argument("id", None)
        resource_id = int(query) if query.isdigit() else 0
        username = self.get_current_user()
        if str(resource_id) in os.getenv("HIDDEN_RESOURCE", "").split(","):
            self.set_status(HTTPStatus.NOT_FOUND)
            return {"status": 0, "info": "资源已隐藏"}
        data = self.instance.get_resource_data(resource_id, username)
        if not data:
            self.ban()
            self.set_status(HTTPStatus.NOT_FOUND)
            data = {}

        return data

    def make_some_fun(self):
        if platform.uname().system == "Darwin":
            return
        referer = self.request.headers.get("referer")
        ip = self.get_real_ip()
        if not referer and self.request.headers.get("origin") != "tauri://localhost":
            cf.ban_new_ip(ip)
            if os.getenv("GIFT"):
                self.set_header("Content-Type", "text/html")
                self.set_header("Content-Encoding", "gzip")
                with open("templates/gift.gz", "rb") as f:
                    return f.read()

    @run_on_executor()
    def search_resource(self):
        if gift := self.make_some_fun():
            return gift

        kw = self.get_query_argument("keyword").lower()
        search_type = self.get_query_argument("type", "default")
        self.set_header("search-engine", "Meilisearch" if os.getenv("MEILISEARCH") else "MongoDB")
        return self.instance.search_resource(kw, search_type)

    @gen.coroutine
    def get(self):
        if self.get_query_argument("id", None):
            resp = yield self.get_resource_data()
        elif self.get_query_argument("keyword", None):
            resp = yield self.search_resource()
        else:
            resp = "error"
        self.write(resp)


class ResourceLatestHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def get_latest(self):
        size = int(self.get_query_argument("size", "100"))
        result = self.instance.get_latest_resource()
        result["data"] = result["data"][:size]
        return result

    @gen.coroutine
    def get(self):
        resp = yield self.get_latest()
        self.write(resp)


class TopHandler(BaseHandler):
    filename = filename

    def get_user_like(self) -> list:
        username = self.get_current_user()
        return self.instance.get_user_like(username)

    def get_most(self) -> list:
        return self.instance.get_most()

    @run_on_executor()
    def get_top_resource(self):
        return self.instance.get_top_resource()

    @gen.coroutine
    def get(self):
        resp = yield self.get_top_resource()
        self.write(resp)


class NameHandler(BaseHandler):
    filename = filename

    @run_on_executor()
    def get_names(self):
        is_readable = self.get_query_argument("human", None)
        return self.instance.get_names(is_readable)

    @gen.coroutine
    def get(self):
        resp = yield self.get_names()
        self.write(resp)


class AdsenseStatusHandler(BaseHandler):
    @run_on_executor()
    def get_adsense_status(self):
        return {"data": os.getenv("HIDE_ADSENSE", "").split(",")}

    @gen.coroutine
    def get(self):
        resp = yield self.get_adsense_status()
        self.write(resp)



File: yyetsweb/handlers/base.py

#!/usr/bin/env python3
# coding: utf-8
import contextlib
import importlib
import json
import logging
import pathlib
from concurrent.futures import ThreadPoolExecutor
from http import HTTPStatus
from pathlib import Path

from tornado import gen, web
from tornado.concurrent import run_on_executor

from databases.base import redis_client
from handlers import cf

index = pathlib.Path(__file__).parent.parent.joinpath("templates", "index.html").as_posix()
filename = Path(__file__).name.split(".")[0]


class BaseHandler(web.RequestHandler):
    key = "user_blacklist"
    filename = filename

    executor = ThreadPoolExecutor(200)

    def __init__(self, application, request, **kwargs):
        super().__init__(application, request, **kwargs)
        self.json = {}
        with contextlib.suppress(ValueError):
            self.json: dict = json.loads(self.request.body)
        class_name = self.__class__.__name__.split("Handler")[0]
        module = importlib.import_module(f"databases.{self.filename}")

        self.instance = getattr(module, class_name, lambda: 1)()
        self.r = redis_client

    def add_tauri(self):
        origin = self.request.headers.get("origin", "")
        allow_origins = ["tauri://localhost", "https://tauri.localhost"]
        if origin in allow_origins:
            self.set_header("Access-Control-Allow-Origin", origin)

    def prepare(self):
        self.add_tauri()
        if self.check_request():
            self.set_status(HTTPStatus.FORBIDDEN)
            self.finish()

    def data_received(self, chunk):
        pass

    def check_request(self):
        ban = self.__ip_check()
        user = self.__user_check()
        result = ban or user
        if result:
            self.ban()
        return result

    def get_real_ip(self):
        x_real = self.request.headers.get("X-Real-IP")
        remote_ip = self.request.remote_ip
        logging.debug("X-Real-IP:%s, Remote-IP:%s", x_real, remote_ip)
        return x_real or remote_ip

    def ban(self):
        ip = self.get_real_ip()
        self.r.incr(ip)
        count = int(self.r.get(ip))
        # ban rule: (count-10)*600
        if count <= 10:
            ex = 120
        else:
            ex = (count - 10) * 600
        if count >= 30:
            cf.ban_new_ip(ip)
        self.r.set(ip, count, ex)
        user = self.get_current_user()
        if user:
            self.r.hincrby(self.key, user)

    def get_current_user(self) -> str:
        username = self.get_secure_cookie("username") or b""
        return username.decode("u8")

    def __user_check(self):
        count = self.r.hget(self.key, self.get_current_user()) or 0
        count = int(count)
        if count >= 20:
            return True

    def __ip_check(self):
        d = self.r.get(self.get_real_ip()) or 0
        if int(d) >= 10:
            return True

    def write_error(self, status_code, **kwargs):
        if status_code in [
            HTTPStatus.FORBIDDEN,
            HTTPStatus.UNAUTHORIZED,
            HTTPStatus.NOT_FOUND,
            HTTPStatus.INTERNAL_SERVER_ERROR,
        ]:
            self.write(str(kwargs.get("exc_info")))


class IndexHandler(BaseHandler):
    @run_on_executor()
    def send_index(self):
        with open(index, encoding="u8") as f:
            html = f.read()
        return html

    @gen.coroutine
    def get(self):
        resp = yield self.send_index()
        self.write(resp)


class NotFoundHandler(BaseHandler):
    def get(self):
        # for react app
        self.render(index)



File: yyetsweb/handlers/oauth.py

#!/usr/bin/env python3
# coding: utf-8
import logging
import os
from pathlib import Path
from urllib.parse import urlencode

import requests
from tornado.auth import GoogleOAuth2Mixin, OAuth2Mixin, TwitterMixin

from handlers.base import BaseHandler

filename = Path(__file__).name.split(".")[0]


class OAuth2Handler(BaseHandler, OAuth2Mixin):
    filename = filename
    _OAUTH_AUTHORIZE_URL = ""
    _OAUTH_ACCESS_TOKEN_URL = ""
    _OAUTH_API_REQUEST_URL = ""

    def add_oauth_user(self, username, unique, source):
        logging.info("User %s login with %s now...", username, source)
        ip = self.get_real_ip()
        browser = self.request.headers["user-agent"]
        result = self.instance.add_user(username, ip, browser, unique, source)
        if result["status"] == "success":
            self.set_secure_cookie("username", username, 365)
        self.redirect("/login?" + urlencode(result))

    def get_authenticated_user(self, client_id: str, client_secret: str, code: str, extra_fields: dict = None):
        args = {"code": code, "client_id": client_id, "client_secret": client_secret}
        if extra_fields:
            args.update(extra_fields)
        return requests.post(
            self._OAUTH_ACCESS_TOKEN_URL,
            headers={"Accept": "application/json"},
            data=args,
        ).json()

    def oauth2_sync_request(self, access_token, extra_fields=None):
        return requests.get(
            self._OAUTH_API_REQUEST_URL,
            headers={"Authorization": f"Bearer {access_token}"},
            params=extra_fields,
        ).json()

    def get_secret(self, settings_key):
        settings = self.settings.get(settings_key)
        client_id = settings.get("key")
        client_secret = settings.get("secret")
        redirect_uri = os.getenv("DOMAIN") + self.request.path
        return client_id, client_secret, redirect_uri


class GitHubOAuth2LoginHandler(OAuth2Handler):
    _OAUTH_AUTHORIZE_URL = "https://github.com/login/oauth/authorize"
    _OAUTH_ACCESS_TOKEN_URL = "https://github.com/login/oauth/access_token"
    _OAUTH_API_REQUEST_URL = "https://api.github.com/user"

    def get(self):
        client_id, client_secret, redirect_uri = self.get_secret("github_oauth")
        code = self.get_argument("code", None)
        if code:
            access = self.get_authenticated_user(client_id, client_secret, code)
            resp = self.oauth2_sync_request(access["access_token"])
            username = resp["login"]
            db_id = resp["id"]
            self.add_oauth_user(username, db_id, "GitHub")
        else:
            self.authorize_redirect(
                redirect_uri=redirect_uri,
                client_id=client_id,
                scope=[],
                response_type="code",
            )


class MSOAuth2LoginHandler(OAuth2Handler):
    _OAUTH_AUTHORIZE_URL = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize"
    _OAUTH_ACCESS_TOKEN_URL = "https://login.microsoftonline.com/common/oauth2/v2.0/token"
    _OAUTH_API_REQUEST_URL = "https://graph.microsoft.com/v1.0/me"

    def get(self):
        client_id, client_secret, redirect_uri = self.get_secret("ms_oauth")
        code = self.get_argument("code", None)
        if code:
            access = self.get_authenticated_user(
                client_id,
                client_secret,
                code,
                {"grant_type": "authorization_code", "redirect_uri": redirect_uri},
            )
            resp = self.oauth2_sync_request(access["access_token"])
            email = resp["userPrincipalName"]
            uid = resp["id"]
            self.add_oauth_user(email, uid, "Microsoft")

        else:
            self.authorize_redirect(
                redirect_uri=redirect_uri,
                client_id=client_id,
                scope=["https://graph.microsoft.com/User.Read"],
                response_type="code",
            )


class GoogleOAuth2LoginHandler(GoogleOAuth2Mixin, OAuth2Handler):
    async def get(self):
        redirect_uri = os.getenv("DOMAIN") + self.request.path
        code = self.get_argument("code", None)
        if code:
            access = await self.get_authenticated_user(redirect_uri=redirect_uri, code=code)
            user = await self.oauth2_request(
                "https://www.googleapis.com/oauth2/v1/userinfo",
                access_token=access["access_token"],
            )
            email = user["email"]
            # Google's email can't be changed
            self.add_oauth_user(email, email, "Google")
        else:
            self.authorize_redirect(
                redirect_uri=redirect_uri,
                client_id=self.settings["google_oauth"]["key"],
                scope=["email"],
                response_type="code",
                extra_params={"approval_prompt": "auto"},
            )


class TwitterOAuth2LoginHandler(TwitterMixin, OAuth2Handler):
    async def get(self):
        if self.get_argument("oauth_token", None):
            user = await self.get_authenticated_user()
            username = user["username"]
            id_str = user["id_str"]
            self.add_oauth_user(username, id_str, "Twitter")
        else:
            await self.authorize_redirect(extra_params={"x_auth_access_type": "read"})


class FacebookAuth2LoginHandler(OAuth2Handler):
    _OAUTH_AUTHORIZE_URL = "https://www.facebook.com/v16.0/dialog/oauth"
    _OAUTH_ACCESS_TOKEN_URL = "https://graph.facebook.com/oauth/access_token"
    _OAUTH_API_REQUEST_URL = "https://graph.facebook.com/me"

    def get(self):
        client_id, client_secret, redirect_uri = self.get_secret("fb_oauth")
        code = self.get_argument("code", None)
        if code:
            access = self.get_authenticated_user(client_id, client_secret, code, {"redirect_uri": redirect_uri})
            resp = self.oauth2_sync_request(access["access_token"], {"fields": "name,id"})
            # Facebook doesn't allow to get email except for business accounts
            uid = resp["id"]
            email = "{}_{}".format(resp["name"], uid)
            self.add_oauth_user(email, uid, "Facebook")

        else:
            self.authorize_redirect(
                redirect_uri=redirect_uri,
                client_id=client_id,
            )



# YYeTsBot

[![build docker image](https://github.com/tgbot-collection/YYeTsBot/actions/workflows/docker.yaml/badge.svg)](https://github.com/tgbot-collection/YYeTsBot/actions/workflows/docker.yaml)
[![Docker Pulls](https://img.shields.io/docker/pulls/bennythink/yyetsbot)](https://hub.docker.com/r/bennythink/yyetsbot)

![](assets/index.png)

👉 前端[在这里](https://github.com/tgbot-collection/YYeTsFE) 👈

# 使用说明

直接发送想要看的剧集名称就可以了，可选分享网页或者链接（ed2k和磁力链接）。


搜索资源时，会按照我预定的优先级（人人影视离线、字幕侠）进行搜索，当然也可以使用命令强制某个字幕组，如 `/yyets_offline 逃避可耻`

## 命令

```
start - 开始使用
help - 帮助
credits - 致谢
ping - 运行状态
settings - 获取公告
zimuxia_online - 字幕侠在线数据  
newzmz_online - new字幕组在线数据 
yyets_offline - 人人影视离线数据
```

# 截图

## 常规搜索

![](assets/1.png)

## 资源分享站截图

本网站永久免费，并且没有任何限制。
![](assets/new_resource.png)

![](assets/2.png)

支持收藏功能，会跨设备同步
![](assets/like.png)

## 指定字幕组搜索

目前只支持YYeTsOffline、ZimuxiaOnline和NewzmzOnline

![](assets/3.png)

# 如何下载磁力和电驴资源？迅雷提示资源敏感

## 电驴资源

请下载使用 [eMule](https://www.emule-project.net/home/perl/general.cgi?l=42) ，然后添加如下两个server list

* [server.met](http://www.server-met.de/)
* [server list for emule](https://www.emule-security.org/serverlist/)

![](assets/emule.jpeg)
速度还可以哦

## 磁力

使用百度网盘、115等离线，或使用utorrent等工具，记得更新下 [tracker list](https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt)
哦

# 小白使用

想要自己留一份资源，但是又不懂编程？ 没关系！目前提供两种方式，请根据自己情况选择
“离线使用” 意味着可以断网使用，但是不会自动更新资源，需要手动更新数据库；“在线应用” 意味着需要有互联网才可以使用。

## 离线  完整运行包

这个版本是新的UI，拥有全部的最新功能。运行在你本地的电脑上，不依赖外界环境。
[参考文档](https://github.com/tgbot-collection/YYeTsBot/blob/master/DEVELOPMENT.md#%E4%B8%80%E9%94%AE%E8%84%9A%E6%9C%AC)

## 离线  一键运行包

一键运行包。拥有比较新的UI，只不过只有最基础的搜索、查看资源的功能。使用方法步骤如下

1. 请到 [GitHub Release](https://github.com/tgbot-collection/YYeTsBot/releases) ，找最新的 `YYeTsBot 离线一键运行包`
2. windows：双击第一步下载的exe文件； macos/Linux，cd到你的目录, `chmod +x yyetsweb ; ./yyetsweb`
3. 程序会自动下载数据库并启动。等到出现启动提示时， 打开浏览器 http://127.0.0.1:8888 就可以看到熟悉的搜索界面啦！

## 在线 原生应用程序

使用tauri封装的网页。使用方法如下

1. 请到 [GitHub Release](https://github.com/tgbot-collection/YYeTsBot/releases) ，找最新的 `YYeTsBot App`
2. windows下载msi，macos下载dmg或tar.gz，Linux下载AppImage或deb（Debian based）
3. 安装后，打开App，就可以看到熟悉的搜索界面啦！

# 开发

## 网站开发

如何部署、参与开发、具体API接口，可以 [参考这个文档](DEVELOPMENT.md)

## Python Library

也可以作为Python Library去调用

`pip3 install yyets`

```
>>> from yyets import YYeTs
>>> yy=YYeTs("逃避")
[2021-09-21 19:22:32 __init__.py:54 I] Fetching 逃避可耻却有用...https://yyets.click/api/resource?id=34812
[2021-09-21 19:22:33 __init__.py:54 I] Fetching 无法逃避...https://yyets.click/api/resource?id=29540
[2021-09-21 19:22:35 __init__.py:54 I] Fetching 逃避者...https://yyets.click/api/resource?id=37089

>>> yy.result
[<yyets.Resource object at 0x10cc7b130>, <yyets.Resource object at 0x10ca0e880>, <yyets.Resource object at 0x10cc7b040>]

>>> for y in yy.result:
        print(y)
    
逃避可耻却有用 - NIGERUHA HAJIDAGA YAKUNITATSU
无法逃避 - Inescapable
逃避者 - Shirkers

>>> yy.result[0].cnname
'逃避可耻却有用'

>>> yy.result[0].list
[{'season_num': '101', 'season_cn': '单剧', 'items': {'APP': [{'ite
```

# Credits

* [人人影视](http://www.zmz2019.com/)
* [追新番](http://www.fanxinzhui.com/)
* [FIX字幕侠](https://www.zimuxia.cn/)
* [new字幕组](https://newzmz.com/)

# 支持我

觉得本项目对你有帮助？你可以通过以下方式表达你的感受：

* 感谢字幕组
* 点一个star🌟和fork🍴
* 宣传，使用，提交问题报告
* 收藏[我的博客](https://dmesg.app/)
* [Telegram Channel](https://t.me/mikuri520)

## 捐助

* [给我买杯咖啡？](https://www.buymeacoffee.com/bennythink)
* [爱发电？](https://afdian.net/@BennyThink)
* [GitHub Sponsor](https://github.com/sponsors/BennyThink)
* [Stripe](https://buy.stripe.com/dR67vU4p13Ox73a6oq)

 <img src="./assets/CNY.png" width = 30%  alt="stripe"  />


# License

[MIT](LICENSE)
