File: src/extractor/danbooru_downloader.py

#coding: utf-8
import downloader
import ree as re
from utils import Downloader, get_max_range, clean_title, get_print, try_n, urljoin, check_alive, LazyUrl, get_ext, limits
from translator import tr_
from urllib.parse import urlparse, parse_qs, quote
import clf2



class Downloader_danbooru(Downloader):
    type = 'danbooru'
    URLS = ['danbooru.donmai.us']
    MAX_CORE = 6
    _name = None
    ACCEPT_COOKIES = [r'(.*\.)?donmai\.us']

    def init(self):
        self.session = clf2.solve(self.url, cw=self.cw)['session'] #5336

    @classmethod
    def fix_url(cls, url):
        if 'donmai.us' in url:
            url = url.replace('http://', 'https://')
        else:
            url = url.replace(' ', '+')
            while '++' in url:
                url = url.replace('++', '+')
            url = f'https://danbooru.donmai.us/posts?tags={quote(url)}'
        if 'donmai.us/posts/' in url:
            url = url.split('?')[0]
        return url.strip('+')

    @property
    def name(self):
        if self._name is None:
            parsed_url = urlparse(self.url)
            qs = parse_qs(parsed_url.query)
            if 'donmai.us/favorites' in self.url:
                id = qs.get('user_id', [''])[0]
                print('len(id) =', len(id), f'"{id}"')
                if not id:
                    raise AssertionError('[Fav] User id is not specified')
                id = f'fav_{id}'
            elif 'donmai.us/explore/posts/popular' in self.url: #4160
                soup = read_soup(self.url, self.session, self.cw)
                id = soup.find('h1').text
            elif 'donmai.us/posts/' in self.url:
                id = re.find(r'donmai\.us/posts/([0-9]+)', self.url, err='no id')
            else:
                tags = qs.get('tags', [])
                tags.sort()
                id = ' '.join(tags)
            if not id:
                id = 'N/A'
            self._name = id
        return clean_title(self._name)

    def read(self):
        self.title = self.name

        if 'donmai.us/posts/' in self.url:
            self.single = True

        imgs = get_imgs(self.url, self.session, self.name, cw=self.cw)

        for img in imgs:
            self.urls.append(img.url)

        self.title = self.name


class Image:
    def __init__(self, id, url, session, cw):
        self._cw = cw
        self.id = id
        self._session = session
        self.url = LazyUrl(url, self.get, self)

    def get(self, url):
        soup = read_soup(url, self._session, self._cw)
        ori = soup.find('li', id='post-option-view-original')
        if ori:
            img = ori.find('a')['href']
        else:
            img = soup.find('li', id='post-info-size').find('a')['href']

        if get_ext(img) == '.zip': #4635
            img = soup.find('section', id='content').find('video')['src']

        img = urljoin(url, img)
        ext = get_ext(img)

        self.filename = f'{self.id}{ext}'
        return img, None


@limits(.5)
def wait(cw):
    check_alive(cw)


def setPage(url, page):
    # Main page
    if re.findall(r'https://[\w]*[.]?donmai.us/?$', url):
        url = f"https://{'danbooru.' if 'danbooru.' in url else ''}donmai.us/posts?page=1"

    # Change the page
    if 'page=' in url:
        url = re.sub('page=[0-9]*', f'page={page}', url)
    else:
        url += f'&page={page}'

    return url


@try_n(12) #4103
def read_soup(url, session, cw, try_=1):
    check_alive(cw)
    wait(cw)
    if try_ > 1:
        session.headers['User-Agent'] = downloader.ua.random #5730
    return downloader.read_soup(url, session=session)


def get_imgs(url, session, title=None, range_=None, cw=None):
    if 'donmai.us/artists' in url:
        raise NotImplementedError()
    if 'donmai.us/posts/' in url:
        id = re.find(r'donmai\.us/posts/([0-9]+)', url, err='no id')
        img = Image(id, url, session, cw)
        return [img]

    print_ = get_print(cw)

    # Range
    max_pid = get_max_range(cw)

    if range_ is None:
        range_ = range(1, 1001)
    print(range_)
    imgs = []
    i = 0
    empty_count = 0
    empty_count_global = 0
    url_imgs = set()
    while i < len(range_):
        check_alive(cw)
        p = range_[i]
        url = setPage(url, p)
        print_(url)
        soup = read_soup(url, session, cw)
        articles = soup.findAll('article')
        if articles:
            empty_count_global = 0
        else:
            empty_count += 1
            if empty_count < 4:
                s = f'empty page; retry... {p}'
                print_(s)
                continue
            else:
                empty_count = 0
                empty_count_global += 1

        if empty_count_global >= 6:
            break

        for article in articles:
            id = article.attrs['data-id']

            #url_img = article.attrs['data-file-url'].strip()
            url_img = urljoin(url, article.find('a', class_='post-preview-link')['href']) #4160

            #print(url_img)
            if url_img not in url_imgs:
                url_imgs.add(url_img)
                img = Image(id, url_img, session, cw)
                imgs.append(img)

        if len(imgs) >= max_pid:
            break

        if cw is not None:
            cw.setTitle(f'{tr_("읽는 중...")}  {title} - {len(imgs)}')
        i += 1

    return imgs[:max_pid]



File: src/extractor/gelbooru_downloader.py

#coding: utf-8
import downloader
import ree as re
from utils import Downloader, urljoin, query_url, get_max_range, get_print, get_ext, clean_title, Session, check_alive, File, clean_url
from translator import tr_
from urllib.parse import quote
import utils


def get_tags(url):
    url = clean_url(url)
    qs = query_url(url)
    if 'page=favorites' in url:
        id = qs.get('id', ['N/A'])[0]
        id = 'fav_{}'.format(id)
    else:
        tags = qs.get('tags', [])
        tags.sort()
        id = ' '.join(tags)
    if not id:
        id = 'N/A'
    return id



class Downloader_gelbooru(Downloader):
    type = 'gelbooru'
    URLS = ['gelbooru.com']
    MAX_CORE = 8
    _name = None
    ACCEPT_COOKIES = [r'(.*\.)?gelbooru\.com']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        if 'gelbooru.com' in url.lower():
            url = url.replace('http://', 'https://')
        else:
            url = url.replace(' ', '+')
            while '++' in url:
                url = url.replace('++', '+')
            url = quote(url)
            url = url.replace('%2B', '+')
            url = 'https://gelbooru.com/index.php?page=post&s=list&tags={}'.format(url)
        return url

    @property
    def name(self):
        if self._name is None:
            tags = get_tags(self.url)
            self._name = tags
        return clean_title(self._name)

    def read(self):
        self.title = self.name

        self.urls += get_imgs(self.url, self.session, self.name, cw=self.cw)

        self.title = self.name


class File_gelbooru(File):
    type = 'gelbooru'
    format = 'id'

    def get(self):
        soup = downloader.read_soup(self['referer'], session=self.session)
        for li in soup.findAll('li'):
            if li.text.strip() == 'Original image':
                break
        else:
            raise Exception('no Original image')
        url = li.find('a')['href']
        d = {
            'id': self['id'],
            }
        return {'url': url, 'name': utils.format('gelbooru', d, get_ext(url))}

    def alter(self):
        return self.get()['url']


def setPage(url, page):
    if 'pid=' in url:
        url = re.sub('pid=[0-9]*', f'pid={page}', url)
    else:
        url += f'&pid={page}'

    if page == 0:
        url = url.replace('&pid=0', '')

    return url


def get_imgs(url, session, title=None, cw=None):
    print_ = get_print(cw)
    url = clean_url(url)
    if 's=view' in url and 'page=favorites' not in url:
        raise NotImplementedError('Not Implemented')

    tags = get_tags(url)
    tags = quote(tags, safe='/')
    tags = tags.replace('%20', '+')
    url = f'https://gelbooru.com/index.php?page=post&s=list&tags={tags}'

    # 2566
    user_id = session.cookies.get('user_id', domain='gelbooru.com')
    if not user_id:
        cookies = {'fringeBenefits': 'yup'}
        session.cookies.update(cookies)
    print_('user_id: {}'.format(user_id))

    # Range
    max_pid = get_max_range(cw)

    imgs = []
    ids = set()
    count_no_imgs = 0
    for p in range(500): #1017
        check_alive(cw)
        url = setPage(url, len(ids))
        print_(url)
        soup = downloader.read_soup(url, session=session)
        posts = soup.findAll(class_='thumbnail-preview')
        imgs_new = []
        for post in posts:
            id_ = int(re.find('[0-9]+', post.find('a')['id'], err='no id'))
            if id_ in ids:
                print('duplicate:', id_)
                continue
            ids.add(id_)
            url_img = urljoin(url, post.find('a')['href'])
            img = File_gelbooru({'id': id_, 'referer': url_img, 'name_hint': f'{id_}{{ext}}'})
            imgs_new.append(img)
        if imgs_new:
            imgs += imgs_new
            count_no_imgs = 0
        else:
            print('no imgs')
            count_no_imgs += 1
            if count_no_imgs > 1:
                print('break')
                break

        if len(imgs) >= max_pid:
            break

        if cw is not None:
            cw.setTitle('{}  {} - {}'.format(tr_('읽는 중...'), title, len(imgs)))

    return imgs[:max_pid]



File: src/extractor/youporn_downloader.py

import downloader
from io import BytesIO
from utils import Downloader, LazyUrl, get_ext, format_filename, try_n
import ytdl
from m3u8_tools import M3u8_stream



class Downloader_youporn(Downloader):
    type = 'youporn'
    single = True
    URLS = ['youporn.com']
    display_name = 'YouPorn'
    ACCEPT_COOKIES = [r'(.*\.)?youporn\.com']

    @classmethod
    def fix_url(cls, url):
        if 'youporn.com' not in url.lower():
            url = 'https://www.youporn.com/watch/{}'.format(url)
        return url

    def read(self):
        video = Video(self.url, cw=self.cw)

        self.urls.append(video.url)
        self.setIcon(video.thumb)

        self.enableSegment()

        self.title = video.title


class Video:
    @try_n(4)
    def __init__(self, url, cw=None):
        ydl = ytdl.YoutubeDL(cw=cw)
        info = ydl.extract_info(url)

        f = info['formats'][-1]
        url_video = f['url']

        ext = get_ext(url_video)
        if ext.lower() == '.m3u8': #6142
            ext = '.mp4'
            url_video = M3u8_stream(url_video, referer=url)

        self.url = LazyUrl(url, lambda _: url_video, self)

        self.url_thumb = info['thumbnails'][0]['url']
        self.thumb = BytesIO()
        downloader.download(self.url_thumb, buffer=self.thumb)
        self.title = info['title']
        self.filename = format_filename(self.title, info['id'], ext)



File: src/extractor/m3u8_downloader.py

from utils import Downloader, LazyUrl, clean_title, Session, get_ext
import utils
from m3u8_tools import playlist2stream, M3u8_stream
import os
from hashlib import md5
from translator import tr_
DEFAULT_N_THREAD = 2


def suitable(url):
    ext = get_ext(url).lower()
    return ext in ('.m3u8', '.mpd')


class Downloader_m3u8(Downloader):
    type = 'm3u8'
    URLS = [suitable]
    single = True
    display_name = 'M3U8'

    @classmethod
    def fix_url(cls, url):
        if '://' not in url:
            url = 'http://' + url
        return url

    def read(self):
        fmt = self.cw.format
        referer = self.url
        if isinstance(fmt, str) and fmt.startswith('referer:'):
            referer = fmt[len('referer:'):]
        self.print_('referer: {}'.format(referer))
        n_thread = DEFAULT_N_THREAD
        if isinstance(fmt, int) and fmt > 0:
            n_thread = fmt
        self.print_('n_thread: {}'.format(n_thread))
        video = Video(self.url, n_thread, referer)
        self.urls.append(video.url)
        self.title = os.path.splitext(os.path.basename(video.filename))[0].replace(b'\xef\xbc\x9a'.decode('utf8'), ':')


class Video:
    def __init__(self, url, n_thread, referer):
        session = Session()
        session.purge([rf'(.*\.)?{utils.domain(url)}'])
        if get_ext(url).lower() == '.mpd':
            def m():
                hdr = session.headers.copy()
                if referer:
                    hdr['Referer'] = referer
                return utils.LiveStream(url, headers=hdr)
            ms = [m]
        else:
            ms = [
                lambda: playlist2stream(url, n_thread=n_thread, session=session),
                lambda: M3u8_stream(url, n_thread=n_thread, session=session),
                ]
        for m in ms:
            try:
                m = m()
                break
            except Exception as e:
                e_ = e
        else:
            raise e_
        if getattr(m, 'live', None) is not None: #5110
            #m = m.live
            hdr = session.headers.copy()
            if referer:
                hdr['Referer'] = referer
            m = utils.LiveStream(url, headers=hdr)
            live = True
        else:
            live = False
        self.url = LazyUrl(url, lambda _: m, self)
        self.title = os.path.splitext(os.path.basename(url).split('?')[0])[0][:50]
        self.id_ = md5(url.encode('utf8')).hexdigest()[:8]
        tail = f' ({self.id_}).mp4'
        if live: #5110
            from datetime import datetime
            tail = ' ' + clean_title(datetime.now().strftime('%Y-%m-%d %H:%M')) + tail
        self.filename = clean_title(self.title, n=-len(tail)) + tail


import selector
@selector.options('m3u8')
def options(urls):
    def f(urls):
        n_thread, ok = utils.QInputDialog.getInt(Downloader.mainWindow, tr_('Set number of threads'), tr_('Number of threads?'), value=DEFAULT_N_THREAD, min=1, max=4, step=1)
        if not ok:
            return
        return n_thread
    def f2(urls):
        referer, ok = utils.QInputDialog.getText(Downloader.mainWindow, tr_('Set a referer'), tr_('Referer?'))
        if not ok:
            return
        return 'referer:'+referer
    return [
        {'text': 'Set the number of threads...', 'format': f},
        {'text': 'Set the referer...', 'format': f2},
        ]



File: src/extractor/afreeca_downloader.py

import downloader
from utils import Soup, Downloader, Session, try_n, format_filename, cut_pair, File, get_print, print_error, json
import ree as re
from io import BytesIO
from m3u8_tools import playlist2stream, M3u8_stream
import errors
import utils
import os


class LoginRequired(errors.LoginRequired):
    def __init__(self, *args):
        super().__init__(*args, method='browser', url='https://login.afreecatv.com/afreeca/login.php')



class Downloader_afreeca(Downloader):
    type = 'afreeca'
    URLS = ['afreecatv.com']
    single = True
    display_name = 'AfreecaTV'
    ACCEPT_COOKIES = [r'(.*\.)?afreecatv\.com']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        if Live_afreeca.is_live(url):
            url = Live_afreeca.fix_url(url)
        return url.rstrip(' /')

    def read(self):
        video = Video({'referer': self.url})
        video.ready(self.cw)
        self.urls.append(video)

        thumb = BytesIO()
        downloader.download(video['url_thumb'], buffer=thumb)
        self.setIcon(thumb)

        self.title = os.path.splitext(video['name'])[0].replace('：', ':')
        self.artist = video['artist']

        if video['live']:
            d = {}
            d['url'] = self.url
            d['title'] = self.artist
            d['thumb'] = thumb.getvalue()
            utils.update_live(d, self.cw)


@try_n(4)
def _get_stream(url_m3u8, session, referer, cw=None):
    print_ = get_print(cw)
    print_(f'_get_stream: {url_m3u8}')
    try:
        stream = playlist2stream(url_m3u8, referer=referer, session=session)
    except Exception as e:
        print_(print_error(e))
        stream = M3u8_stream(url_m3u8, referer=referer, session=session)
    return stream



class Video(File):
    type = 'afreeca'
    _live_info = None

    def get(self):
        print_ = get_print(self.cw)
        url, session = self['referer'], self.session
        if session is None:
            session = Session()
            session.purge('afreeca')

        html = downloader.read_html(url, session=session)
        if "document.location.href='https://login." in html:
            raise LoginRequired()
        if len(html) < 2000:
            alert = re.find(r'''alert\(['"](.+?)['"]\)''', html)
            if alert:
                raise LoginRequired(alert)
        soup = Soup(html)

        url_thumb = soup.find('meta', {'property': 'og:image'}).attrs['content']
        print_('url_thumb: {}'.format(url_thumb))

        vid = re.find('/player/([0-9]+)', url)
        if vid is None: # live
            bid = re.find('afreecatv.com/([^/]+)', url, err='no bid')

            url_api = f'https://st.afreecatv.com/api/get_station_status.php?szBjId={bid}'
            r = session.post(url_api, headers={'Referer': url})
            d = json.loads(r.text)
            artist = d['DATA']['user_nick']
            if self._live_info is not None:
                self._live_info['title'] = artist

            url_api = f'https://live.afreecatv.com/afreeca/player_live_api.php?bjid={bid}'
            #bno = re.find('afreecatv.com/[^/]+/([0-9]+)', url, err='no bno')
            bno = re.find(r'nBroadNo\s=\s([0-9]+)', html, err='no bno') #6915
            r = session.post(url_api, data={'bid': bid, 'bno': bno, 'type': 'aid', 'pwd': '', 'player_type': 'html5', 'stream_type': 'common', 'quality': 'master', 'mode': 'landing', 'from_api': '0'}, headers={'Referer': url})
            d = json.loads(r.text)
            res = d['CHANNEL'].get('RESULT')
            print_(f'result: {res}')
            if res == -6:
                raise LoginRequired()
            aid = d['CHANNEL']['AID']

            data = {}
            data['title'] = soup.find('meta', {'property': 'og:title'})['content'].strip()
            data['files'] = [{'file': f'https://pc-web.stream.afreecatv.com/live-stm-16/auth_master_playlist.m3u8?aid={aid}'}]
            data['writer_nick'] = artist
            data['live'] = True
        elif f'{vid}/catch' in url: #6215
            url_api = 'https://api.m.afreecatv.com/station/video/a/catchview'
            r = session.post(url_api, data={'nPageNo': '1', 'nLimit': '10', 'nTitleNo': vid}, headers={'Referer': url})
            try:
                s = cut_pair(r.text)
                d = json.loads(s)
            except Exception as e:
                print_(r.text)
                raise e
            data = d['data'][0]
        else:
            url_api = 'https://api.m.afreecatv.com/station/video/a/view'
            r = session.post(url_api, data={'nTitleNo': vid, 'nApiLevel': '10', 'nPlaylistIdx': '0'}, headers={'Referer': url})
            try:
                s = cut_pair(r.text)
                d = json.loads(s)
            except Exception as e:
                print_(r.text)
                raise e
            data = d['data']

        title = data.get('full_title') or data['title']
        artist = data.get('copyright_nickname') or data.get('original_user_nick') or data['writer_nick']

        if data.get('adult_status') == 'notLogin':
            raise LoginRequired(title)

        urls_m3u8 = []
        for file in data['files']:
            if file.get('quality_info'):
                file = file['quality_info'][0]['file']
            else:
                file = file['file']
            urls_m3u8.append(file)
        print_(f'urls_m3u8: {len(urls_m3u8)}')

        if data.get('live'):
            hdr = session.headers.copy()
            hdr['Referer'] = url
            stream = utils.LiveStream(urls_m3u8[0], headers=hdr, cw=self.cw)
        else:
            streams = []
            for url_m3u8 in urls_m3u8:
                try:
                    stream = _get_stream(url_m3u8, session, url, cw=self.cw)
                except Exception as e:
                    print_(print_error(e))
                    continue #2193
                streams.append(stream)
            for stream in streams[1:]:
                streams[0] += stream
            stream = streams[0]

        live = data.get('live') or False
        return {'url': stream, 'title': title, 'name': format_filename(title, vid, '.mp4', artist=artist, live=live), 'url_thumb': url_thumb, 'artist': artist, 'live': live}



class Live_afreeca(utils.Live):
    type = 'afreeca'

    @classmethod
    def is_live(cls, url):
        return bool(re.match(r'https?://(play|bj).afreecatv.com/([^/?#]+)', url)) and url.strip('/').count('/') <= 4

    @classmethod
    def fix_url(cls, url):
        bj = re.find(r'https?://(play|bj).afreecatv.com/([^/?#]+)', url)[1]
        return f'https://play.afreecatv.com/{bj}'

    @classmethod
    def check_live(cls, url, info=None):
        try:
            video = Video({'referer': url})
            video._live_info = info
            video.ready(None)
            return True
        except Exception as e:
            print(e)
            return False



File: src/extractor/naverpost_downloader.py

# coding: UTF-8
# title: Download naver post image
# author: SaidBySolo
# comment: 네이버 포스트의 이미지를 다운로드합니다

"""
MIT License

Copyright (c) 2020 SaidBySolo

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import codecs
import json
import re

from distutils.util import strtobool
from typing import Any, Iterator, List
from urllib.parse import ParseResult, urlparse, parse_qs

import requests
from bs4 import BeautifulSoup
import clf2
import page_selector

from utils import Downloader, Soup, clean_title


class Page:
    def __init__(self, title, url) -> None:
        self.title = clean_title(title)
        self.url = url



class DownloaderNaverPost(Downloader):
    type = "naverpost"  # 타입
    URLS = ["m.post.naver.com", "post.naver.com"]

    def init(self) -> None:
        self.parsed_url = urlparse(self.url)  # url 나눔
        self.soup = get_soup(self.url)

    @property
    def client(self):
        return Client(self.parsed_url, self.soup)

    def read(self):
        if self.client.single:
            self.title = self.client.title
            posts = self.client.posts
        else:
            raise NotImplementedError

        for img_link in img_src_generator(posts):
            self.urls.append(img_link)


# https://github.com/KurtBestor/Hitomi-Downloader/blob/master/src/extractor/manatoki_downloader.py#L106 참고
@page_selector.register("naverpost")
def f(url, win):
    client = Client(urlparse(url), get_soup(url, win=win))
    return [
        page for page_list in client.posts for page in page_list
    ]  # 2차원 리스트 -> 1차원 리스트


# https://github.com/KurtBestor/Hitomi-Downloader/blob/master/src/extractor/manatoki_downloader.py#L84 참고
def get_soup(url: str, win=None) -> BeautifulSoup:
    res = clf2.solve(url, win=win)
    return Soup(res["html"])


# 페이지 파싱에서 사용되는 파서
def page_soup(url: str) -> BeautifulSoup:
    get_html_regex = re.compile(r"\"html\"\:(.+)(\n|\s)\}")
    response = requests.get(url)
    like_html = get_html_regex.search(response.text)[1]
    html = decode_escapes(like_html).replace(r"\/", "/")
    return Soup(html)


# HTML5 data-* 속성이 사용됨.
def get_img_data_linkdatas(soup: Any) -> Iterator[str]:
    a_elements = soup.find_all("a", {"data-linktype": "img"})  # 링크 타입이 img인것만 전부 찾음
    for a_element in a_elements:
        yield a_element["data-linkdata"]


def img_src_generator(linkdatas: Iterator[str]) -> Iterator[str]:
    for linkdata in linkdatas:
        data = json.loads(linkdata)
        if data.get("linkUse") is None:
            yield data["src"]  # 제네레이터
        else:
            if not strtobool(data["linkUse"]):
                yield data["src"]


# https://stackoverflow.com/a/24519338 참고
def decode_escapes(like_html: str) -> str:
    escape_sequence_regex = re.compile(
        r"""
        ( \\U........      # 8-digit hex escapes
        | \\u....          # 4-digit hex escapes
        | \\x..            # 2-digit hex escapes
        | \\[0-7]{1,3}     # Octal escapes
        | \\N\{[^}]+\}     # Unicode characters by name
        | \\[\\'"abfnrtv]  # Single-character escapes
        )""",
        re.UNICODE | re.VERBOSE,
    )

    return escape_sequence_regex.sub(
        lambda match: codecs.decode(match.group(0)), like_html
    )


# 제목
class Title:
    def __init__(self, soup: Any):
        self.soup = soup

    def get_profile_title(self) -> str:
        profile_name = self.soup.find("p", class_="nick_name").find(
            "span", class_="name"
        )  # 프로필 닉네임
        return clean_title(profile_name.text)  # 닉네임으로만

    def get_series_title(self) -> str:
        series_name = self.soup.find("h2", class_="tit_series").find(
            "span", class_="ell"
        )  # 시리즈 제목
        author = self.soup.find("div", class_="series_author_wrap").find(
            "strong", class_="ell1"
        )  # 작성자
        return clean_title(f"{series_name.text} ({author.text})")  # 무난하게 붙임

    def get_title(self) -> str:
        title = self.soup.find("h3", class_="se_textarea")  # 포스트 제목
        author = self.soup.find("span", class_="se_author")  # 작성자
        return clean_title(f"{title.text.replace(' ', '')} ({author.text})")  # 무난하게 붙임


# 총 포스트 수
class Total:
    def __init__(self, soup: Any) -> None:
        self.soup = soup

    # 0: 팔로워 1: 팔로잉 2: 포스트 3: 좋아요한글
    def get_total_post(self) -> int:
        profile_info = self.soup.find("div", class_="expert_num_info")  # 프로필 정보
        total_post_element = profile_info.find_all("li", class_="inner")[2]
        return int(total_post_element.find("span", class_="num").text)  # 총몇개인지만 리턴

    # 0: 포스트 1: 팔로워
    def get_series_total_post(self) -> int:
        series_info = self.soup.find("div", class_="series_follow_area")  # 시리즈 정보
        total_post_element = series_info.find_all("a")[0]
        return int(total_post_element.find("em").text)  # 총몇개인지만 리턴


class UrlGenerator:
    def __init__(self, parsed_url: ParseResult, total_count: int) -> None:
        self.parsed_url = parsed_url
        self.count = (
            round(total_count / 20) + 1
            if not (total_count / 20).is_integer()
            else round(total_count / 20)
        )

    def all_post_url_generator(self) -> Iterator[str]:
        query = parse_qs(self.parsed_url.query)
        for i in range(self.count):
            new_url_query = f"?memberNo={query['memberNo'][0]}&fromNo={i + 1}"
            url = f"https://{self.parsed_url.netloc}/async{self.parsed_url.path}{new_url_query}"
            yield url

    def all_series_url_generator(self) -> Iterator[str]:
        query = parse_qs(self.parsed_url.query)
        for i in range(self.count):
            new_url_query = f"?memberNo={query['memberNo'][0]}&seriesNo={query['seriesNo'][0]}&fromNo={i + 1}"
            url = f"https://{self.parsed_url.netloc}/my/series/detail/more.nhn{new_url_query}"
            yield url


# 여기서 페이지 리스트 만듬
class PostPage:
    def __init__(self, soup: Any):
        self.soup = soup

    def all_post_page_generator(self) -> Iterator[List[Page]]:
        titles = self.soup.find_all("strong", class_="tit_feed ell")
        link_elements = self.soup.find_all("a", class_="link_end", href=True)

        page = [
            Page(title.text.replace(" ", ""), link_element["href"])
            for link_element, title in zip(link_elements, titles)
        ]

        yield page[::-1]

    def all_series_page_generator(self) -> Iterator[List[Page]]:
        titles = [
            element.find("span")
            for element in self.soup.find_all("div", class_="spot_post_name")
        ]
        link_elements = self.soup.find_all("a", class_="spot_post_area", href=True)

        page = [
            Page(title.text.replace(" ", ""), link_element["href"])
            for link_element, title in zip(link_elements, titles)
        ]

        yield page[::-1]


# 필요한 클래스 전부 상속후 편하게 쓸수있게 만듬
class Client(Title, Total, UrlGenerator):
    def __init__(self, parsed_url: ParseResult, soup: BeautifulSoup):
        Title.__init__(self, soup)
        Total.__init__(self, soup)

        if parsed_url.path.startswith("/viewer"):
            self.title = self.get_title()
            self.posts = get_img_data_linkdatas(self.soup)
            self.single = True

        elif parsed_url.path.startswith("/my.nhn"):
            UrlGenerator.__init__(self, parsed_url, self.get_total_post())
            self.title = self.get_profile_title()
            self.posts = self.all_post_url_generator()
            self.single = False

        elif parsed_url.path.startswith("/my/series"):
            UrlGenerator.__init__(self, parsed_url, self.get_series_total_post())
            self.title = self.get_series_title()
            self.posts = self.all_series_url_generator()
            self.single = False

        else:
            raise Exception("유효하지 않습니다.")



File: src/extractor/misskey_downloader.py

from utils import Downloader, Session, clean_title, get_ext, check_alive, tr_, try_n, File, get_max_range, limits
import downloader
import ree as re
from datetime import datetime
import utils
import errors
DOMAIN = 'misskey.io'
SUBFOLDER = True


class File_misskey(File):
    type = 'misskey'
    format = '[date] id_ppage'


def get_file(nid, url, referer, session, p, time):
    ext = get_ext(url) or downloader.get_ext(url, session, referer)
    d = {
        'date': time,
        'id': nid,
        'page': p,
        }
    filename = utils.format('misskey', d, ext)
    info = {'name': filename, 'url': url, 'referer': referer}
    return File_misskey(info)


def get_time(note):
    ds = note['createdAt']
    time = datetime.strptime(ds.split('.')[0], '%Y-%m-%dT%H:%M:%S')
    time = (time-datetime(1970,1,1)).total_seconds()
    return time


class Downloader_misskey(Downloader):
    type = 'misskey'
    URLS = [f'{DOMAIN}/notes/', f'{DOMAIN}/@']
    display_name = 'Misskey'
    ACCEPT_COOKIES = [rf'(.*\.)?{DOMAIN}']
    MAX_CORE = 8

    @classmethod
    def fix_url(cls, url):
        if DOMAIN.lower() in url.lower() and '://' not in url:
            url = 'https://' + url
        if url.startswith('@'):
            url = f'https://{DOMAIN}/{url}'
        return url

    def init(self):
        self.session = Session()
        if f'{DOMAIN}/notes/' in self.url:
            raise errors.Invalid(tr_('개별 다운로드는 지원하지 않습니다: {}').format(self.url))

    @try_n(4, sleep=5)
    @limits(2)
    def call(self, path, payload):
        token = self.session.cookies.get('token', domain=DOMAIN)
        url_api = f'https://{DOMAIN}/api/{path}'
        if token:
            payload['i'] = token
        r = self.session.post(url_api, json=payload)
        d = r.json()
        if isinstance(d, dict):
            err = d.get('error')
            if err:
                raise errors.Invalid(err['message'])
        return d

    def read(self):
        nid = re.find(rf'{DOMAIN}/notes/([^/]+)', self.url)
        if nid:
            self.single = True
            data = {'noteId':nid,
                    }
            note = self.call('notes/show', data)
            username = note['user']['username']
            self.artist = note['user']['name'] or username
            host = note['user']['host']
            if host:
                username += f'@{host}'
            self.title = f'{clean_title(self.artist)} (misskey_@{username})'
            time = get_time(note)
            for file in note['files']:
                file = get_file(note['id'], file['url'], self.url, self.session, len(self.urls), time)
                if SUBFOLDER:
                    file['name'] = self.title + '/' + file['name']
                self.urls.append(file)
        else:
            username = re.find(rf'{DOMAIN}/@([a-zA-Z0-9_@\.]+)', self.url, err='no username')
            if '@' in username:
                username, host = username.split('@')
            else:
                host = None
            data = {"username":username,
                    "host":host,
                    }
            d = self.call('users/show', data)
            username = d['username']
            self.artist = d['name'] or username
            host = d['host'] or None
            if host:
                username += f'@{host}'
            uid = d['id']
            self.title = title = f'{clean_title(self.artist)} (misskey_@{username})'
            untilId = None
            nids = set()
            n = get_max_range(self.cw)
            while check_alive(self.cw):
                data = {"userId":uid,
                        "limit":30,
                        }
                if untilId:
                    data["untilId"] = untilId
                d = self.call('users/notes', data)
                if not d:
                    break
                for note in d:
                    nid = note['id']
                    if nid in nids:
                        continue
                    nids.add(nid)
                    time = get_time(note)
                    url_note = f'https://{DOMAIN}/notes/{nid}'
                    for p, file in enumerate(note['files']):
                        file = get_file(note['id'], file['url'], url_note, self.session, p, time)
                        self.urls.append(file)
                    untilId = nid
                self.cw.setTitle(f'{tr_("읽는 중...")}  {title} - {len(self.urls)}')
                if len(self.urls) >= n:
                    break
            self.title = title



File: src/extractor/youtube_downloader.py

#coding: utf-8
import ytdl
import downloader
import downloader_v3
from error_printer import print_error
from timee import sleep
import ree as re
from utils import urljoin, Downloader, try_n, get_print, filter_range, compatstr, uuid, get_max_range, format_filename, get_resolution, get_abr, Session, fix_dup, File, clean_title
import ffmpeg
import constants
import os
import utils
from translator import tr, tr_
from datetime import datetime
import threading
from putils import DIR
import errors
MODE = 'query'
utils.TOKENS['youtube'] = ['title', 'id', 'artist', 'date'] + utils.ADD_TOKENS


def print_streams(streams, cw):
    print_ = get_print(cw)

    for stream in streams:
        print_(f'{"LIVE " if stream.live else ""}[{stream.resolution}][{stream.fps}fps][{stream.abr_str}{"(fixed)" if stream.abr_fixed else ""}][{stream.tbr}] {stream.subtype} [{stream.video_codec} / {stream.audio_codec}] ─ {stream.format}')
    print_('')


class Video(File):
    type = 'youtube'
    vcodec = None
    filename0 = None
    chapters = None
    _yt = None
    _thumb = None

    @property
    def yt(self):
        print_ = get_print(self.cw)
        if self._yt is None:
            for try_ in range(4):
                try:
                    self._yt = ytdl.YouTube(self['referer'], cw=self.cw)
                    break
                except errors.Retry as e:
                    raise e
                except Exception as e:
                    e_ = e
                    s = print_error(e)
                    print_('### youtube retry...\n{}'.format(s))
                    sleep(try_, self.cw)
            else:
                raise e_
        return self._yt

    def thumb(self):
        if self._thumb is None:
            self.thumb_url, self._thumb = ytdl.download_thumb(self.yt.thumbnail_url, self.cw, self.session)
        self._thumb.seek(0)
        return self._thumb

    def get(self):
        type = self['type']
        only_mp4 = self['only_mp4']
        audio_included = self['audio_included']
        max_res = self['max_res']
        max_abr = self['max_abr']
        cw = self.cw
        session = self.session
        url = self['referer']
        print_ = get_print(cw)

        print('max_res: {}'.format(max_res))
        yt = self.yt

        if utils.ui_setting.chapterMarkerCheck.isChecked():
            self.chapters = yt.info.get('chapters')

        streams = yt.streams.all()
        print_streams(streams, cw)

        #3528
        time = datetime.strptime(yt.info['upload_date'], '%Y%m%d')
        if utils.ui_setting.youtubeMtimeCheck.isChecked(): #6092
            self.utime = (time-datetime(1970,1,1)).total_seconds()
        print_('utime: {}'.format(self.utime))

        if type == 'video':
            streams[:] = [stream for stream in streams if stream.video_codec is not None]
            # Only mp4
            if only_mp4:
                streams_ = list(streams)
                streams[:] = []
                for stream in streams_:
                    if stream.subtype == 'mp4':
                        streams.append(stream)

            # Audio included; Non-HD
            if audio_included:
                streams_ = list(streams)
                streams[:] = []
                for stream in streams_:
                    if stream.audio_codec is not None:
                        streams.append(stream)

            # Maximum resolution
            streams_ = list(streams)
            streams[:] = []
            for stream in streams_:
                if stream.resolution is None:
                    continue
                res = int(stream.resolution.replace('p',''))
                if max_res is None or res <= max_res:
                    streams.append(stream)
            def key(stream):
                fps = stream.fps
                vc = stream.video_codec
                if vc:
                    vc = vc.lower().split('.')[0].lower()
                if vc == 'av01':
                    vc = 'av1'
                if vc == 'vp09':
                    vc = 'vp9'
                try:
                    i = constants.CODECS_PRI.index(vc)
                except ValueError:
                    i = 999
                pr = 'premium' in stream.format.lower() #6350
                return not pr, i, -fps, -stream.tbr
            streams = sorted(streams, key=key) #6079
            print_('')
        elif type == 'audio':
            streams[:] = [stream for stream in streams if stream.abr]
            # Maximum abr
            abrs = [stream.abr for stream in streams]
            max_abr = min(max(abrs), max_abr)
            streams_ = list(streams)
            streams[:] = []
            for stream in streams_:
                if stream.abr is None:
                    continue
                abr = stream.abr
                if max_abr is None or abr >= max_abr:
                    streams.append(stream)
            #'''
        else:
            raise Exception('type "{}" is not supported'.format(type))

        # Pick the best
        while streams:
            if type == 'video':
                ress = [int_(stream.resolution.replace('p', '')) for stream in streams]
                m = max(ress)
                prefer_format = None#'mp4'
            elif type == 'audio':
                ress = [stream.abr for stream in streams]
                m = min(ress)
                prefer_format = 'webm'
            print('Resolutions:', ress)
            stream_final = None
            for stream, res in zip(streams, ress):
                if res == m:
                    if type == 'video':
                        foo = (stream_final is not None) and (stream_final.audio_codec is None) and bool(stream.audio_codec) and stream_final.fps <= stream.fps #6911
                    elif type == 'audio':
                        foo = False
                    if stream_final is None or (foo or (stream_final.subtype.lower()!=prefer_format and stream.subtype.lower()==prefer_format)):
                        #print(foo)
                        print_('# stream_final')
                        print_streams([stream], cw)
                        stream_final = stream

            ok = downloader.ok_url(stream_final.url, referer=url, session=session) if isinstance(stream_final.url, str) else True
            if ok:
                break
            else:
                print_('stream is not valid')
                streams.remove(stream_final)
        else:
            if type == 'audio' and max_abr > 0:
                self['max_abr'] = 0
                return self.get(url) # 1776
            raise Exception('No videos')

        stream = stream_final

##        if stream.video_codec and stream_final.video_codec.lower().startswith('av'):
##            self.vcodec = 'h264'

        self.id = yt.video_id
        self.stream = stream
        self.username = yt.info['uploader']
        self.stream_audio = None
        self.audio = None
        self.thumb_url = None

        if type == 'audio' and 'DASH' in self.stream.format:
            self.stream.setDashType('audio')

        # Audio
        if type=='video' and stream.audio_codec is None:
            print('audio required')
            streams = [stream for stream in yt.streams.all() if stream.abr]
            print_streams(streams, cw)
            # only mp4; https://github.com/KurtBestor/Hitomi-Downloader/issues/480
            def isGood(stream):
                return stream.audio_codec.lower().startswith('mp4')
            streams_good = [stream for stream in streams if isGood(stream)]
            if streams_good:
                streams = streams_good
                print_streams(streams, cw)
            # only audio?
            if any(stream.resolution is None for stream in streams):
                streams = [stream for stream in streams if stream.resolution is None]
                print_streams(streams, cw)
            def key(stream):
                abr = stream.abr
                format_note = stream.video.get('format_note')
                if format_note and 'original' in format_note.lower():
                    org = 0
                else:
                    org = 1
                lang = stream.video.get('language')
                if lang and constants.ALANG:
                    match_full = lang.lower().startswith(constants.ALANG)
                    match_part = lang.lower().startswith(constants.ALANG.split('-')[0])
                    if match_full or match_part:
                        lang = -1 if match_full else 0
                    else:
                        lang = 1
                else:
                    lang = 1
                return lang, org, -abr
            streams = sorted(streams, key=key) #6332
            best_audio = streams[0]
            print_streams([best_audio], cw)
            self.stream_audio = best_audio
            if 'DASH' in self.stream_audio.format:
                self.stream_audio.setDashType('audio')
            self.audio = best_audio.url
            if callable(self.audio):
                self.audio = self.audio()

        #
        _url = self.stream.url
        if callable(_url):
            _url = _url()
        title = yt.title
        #soup = Soup(yt.watch_html)
        #title =  soup.title.text.replace('- YouTube', '').strip()
        self.title = title
        ext = '.' + self.stream.subtype

        #6425
        d = {}
        v = self.stream.video
        if type != 'audio':
            d['width'] = v['width']
            d['height'] = v['height']
        tokens = ['fps',  'vcodec', 'acodec', 'audio_channels', 'language', 'vbr', 'abr', 'tbr']
        for token in tokens:
            value = v.get(token)
            if isinstance(value, str):
                value = clean_title(value)
            d[token] = value
        if self.stream_audio:
            v = self.stream_audio.video
            for token in tokens:
                value = v.get(token)
                if isinstance(value, str):
                    value = clean_title(value)
                _ = d.get(token)
                if not _ or _ == 'none':
                    d[token] = value

        filename = format_filename(title, yt.video_id, ext, artist=yt.info['uploader'], date=None if self.stream.live else time, d=d, live=self.stream.live) #4953, #5529
        if cw:
            filename = fix_dup(filename, cw.downloader.cache_filenames) #6235
        print_(f'filename: {filename}')

        if type == 'audio':
            self.filename0 = filename
            filename = f'{uuid()}_audio.tmp' #4776

        print_('Resolution: {}'.format(stream.resolution))
        print_('Codec: {} / {}'.format(stream.video_codec, stream.audio_codec))
        print_('Abr: {}'.format(stream.abr))
        print_('Subtype: {}'.format(stream.subtype))
        print_('FPS: {}\n'.format(stream.fps))

        if self.audio is not None: #5015
            def f(audio):
                print_('Download audio: {}'.format(audio))
                path = os.path.join(DIR, f'{uuid()}_a.tmp')
                if cw is not None:
                    cw.trash_can.append(path)
                if constants.FAST:
                    downloader_v3.download(audio, session=session, chunk=1024*1024, n_threads=2, outdir=os.path.dirname(path), fileName=os.path.basename(path), customWidget=cw, overwrite=True, mode=MODE)
                else:
                    downloader.download(audio, session=session, outdir=os.path.dirname(path), fileName=os.path.basename(path), customWidget=cw, overwrite=True)
                self.audio_path = path
                print_('audio done')
            self.thread_audio = threading.Thread(target=f, args=(self.audio,), daemon=True)
            self.thread_audio.start()

        return {'url': _url, 'name': filename}

    def pp(self, filename, i=0):
        cw = self.cw
        print_ = get_print(cw)
        ui_setting = utils.ui_setting
        ext = os.path.splitext(filename)[1].lower()
        if not os.path.isfile(filename):
            print('no file: {}'.format(filename))
            return

        filename_new = filename
        if self['type'] == 'video' and (self.audio is not None or ext != '.mp4') and not self.stream.live: # UHD or non-mp4
            if self.audio is not None: # merge
                self.thread_audio.join()
                ext, out = ffmpeg.merge(filename, self.audio_path, cw=cw, vcodec=self.vcodec)
                #print(out)
                name, ext_old = os.path.splitext(filename)
                if ext_old.lower() != ext.lower():
                    print_('rename ext {} --> {}'.format(ext_old, ext))
                    filename_new = '{}{}'.format(name, ext)
                    if os.path.isfile(filename_new):
                        os.remove(filename_new)
                    os.rename(filename, filename_new)
            else: # convert non-mp4 video -> mp4
                name, ext_old = os.path.splitext(filename)
                filename_new = '{}.mp4'.format(name)
                print_('Convert video: {} -> {}'.format(filename, filename_new))
                ffmpeg.convert(filename, filename_new, cw=cw)
        elif self['type'] == 'audio' and ext != '.mp3': # convert non-mp3 audio -> mp3
            name, ext_old = os.path.splitext(filename)
            filename_new = '{}.mp3'.format(name)
            ffmpeg.convert(filename, filename_new, '-shortest -preset ultrafast -b:a {}k'.format(get_abr()), cw=cw)

        if self.filename0 and os.path.basename(filename_new) != self.filename0: #4776
            filename0 = utils.fix_enumerate(self.filename0, i, cw)
            filename_old = filename_new
            ext = '.mp4' if self['type'] == 'video' else '.mp3'
            filename_new = os.path.join(os.path.dirname(filename_old), os.path.splitext(filename0)[0]+ext)
            print_(f'rename: {filename_old} -> {filename_new}')
            if filename_old != filename_new:
                if not os.path.exists(os.path.dirname(filename_new)):
                    os.makedirs(os.path.dirname(filename_new))
                if os.path.isfile(filename_new):
                    os.remove(filename_new)
                os.rename(filename_old, filename_new)

        if self['type'] == 'audio' and ui_setting.albumArt.isChecked():
            try:
                ffmpeg.add_cover(filename_new, self.thumb(), {'artist':self.yt.info['uploader'], 'title':self.title}, cw=cw)
            except Exception as e:
                s = print_error(e)
                print_(s)

        if self.chapters and self['type'] == 'video': #6085
            try:
                chapters = []
                for chapter in self.chapters:
                    chapter = ffmpeg.Chapter(chapter['title'], chapter['start_time'], chapter['end_time'])
                    chapters.append(chapter)
                ffmpeg.add_chapters(filename_new, chapters, cw=cw)
            except Exception as e:
                s = print_error(e)
                print_(s)

        return filename_new

    def pp_always(self, filename):
        cw = self.cw
        print_ = get_print(cw)
        if utils.ui_setting.thumbCheck.isChecked():
            import filetype
            s = self.thumb().getvalue()
            ext = filetype.guess(s)
            if ext is None:
                raise Exception('unknown ext')
            filename_thumb = os.path.splitext(filename)[0] + '.' + ext.extension
            print_(f'filename_thumb: {filename_thumb}')
            with open(filename_thumb, 'wb') as f:
                f.write(s)
            cw.imgs.append(filename_thumb)
            cw.dones.add(os.path.abspath(filename_thumb))
        if utils.ui_setting.subtitle.isChecked():
            self.subs = self.yt.subtitles
            utils.pp_subtitle(self, filename, cw)
        return filename


def get_id(url):
    id_ = re.find(r'youtu.be/([0-9A-Za-z-_]{10,})', url) or re.find(r'[?&]v=([0-9A-Za-z-_]{10,})', url) or re.find(r'/(v|embed|shorts|live)/([0-9A-Za-z-_]{10,})', url) or re.find(r'%3Fv%3D([0-9A-Za-z-_]{10,})', url) #5679
    if isinstance(id_, tuple):
        id_ = id_[-1]
    return id_


class Downloader_youtube(Downloader):
    type = 'youtube'
    single = True
    yt_type = None
    URLS = ['youtube.co', 'youtu.be', 'yewtu.be']
    lock = True
    display_name = 'YouTube'
    keep_date = True #3528
    __format = {}
    ACCEPT_COOKIES = [r'.*(youtube|youtu\.be|google).*']
    atts = ['cache_filenames']

    def init(self):
        self.cache_filenames = {}
        format = self.cw.format
        if format:
            if isinstance(format, str):
                ext_result = format
            elif isinstance(format, dict):
                ext_result = format['format']
                self.__format = format
            else:
                raise NotImplementedError(format)
        else:
            ext_result = default_option()
            self.cw.format = ext_result

        if ext_result in ['mp4', 'mkv', '3gp']:
            self.yt_type = 'video'
        else:
            self.yt_type = 'audio'
            self.cw.setMusic(True)
        self.session = Session()

    @classmethod
    def fix_url(cls, url): #2033
        url = url.replace('yewtu.be', 'youtube.com')
        if not re.match('https?://.+', url, re.I):
            url = 'https://www.youtube.com/watch?v={}'.format(url)

        id_ = get_id(url)
        if id_: #6485
            url = 'https://www.youtube.com/watch?v={}'.format(id_)

        for header in ['channel', 'user', 'c']: #5365, #5374
            tab = re.find(rf'/{header}/[^/]+/?(.+)?', url, re.I)
            if tab == 'playlists':
                url = re.sub(rf'(/{header}/[^/]+/?)(.+)?', r'\1', url, flags=re.I)
                tab = ''
            if tab in ['', 'featured'] and '/{}/'.format(header) in url.lower():
                username = re.find(r'/{}/([^/\?]+)'.format(header), url, re.I)
                url = urljoin(url, '/{}/{}/videos'.format(header, username))
        m = re.find(r'youtube.com/(@[^/]+)/?(.+)?', url, re.I)
        if m and m[1] in ['', 'featured']: #6129
            url = urljoin(url, f'/{m[0]}/videos')
        return url.strip('/')

    @classmethod
    def key_id(cls, url):
        return get_id(url) or url

    @classmethod
    def is_channel_url(cls, url):
        if '/channel/' in url or '/user/' in url or '/c/' in url:
            return True
        if ''.join(url.split('/')[3:4]).startswith('@'):
            return not url.lower().endswith('/live')
        return False

    def read(self):
        cw = self.cw
        if self.yt_type == 'video':
            res = self.__format.get('res', get_resolution())
            info = get_videos(self.url, self.session, type=self.yt_type, max_res=res, only_mp4=False, audio_included=not True, cw=cw)
        else:
            abr = self.__format.get('abr', get_abr())
            info = get_videos(self.url, self.session, type=self.yt_type, max_abr=abr, cw=cw)
        videos = info['videos']

        if not videos:
            raise Exception('No videos')

        self.enableSegment(overwrite=True)
        self.cw.v3['mode'] = MODE

        # first video must be valid
        while videos:
            video = videos[0]
            try:
                video.ready(cw)
                break
            except Exception as e:
                e_ = e
                self.print_(print_error(e))
                videos.remove(video)
        else:
            raise e_

        if info['type'] != 'single':
            video = self.process_playlist(info['title'], videos)
        else:
            self.urls.append(video)
            self.title = os.path.splitext(video.filename0 or video['name'])[0].replace('：', ':') #4776
            if video.stream.live:
                self.lock = False

        self.artist = video.username
        self.setIcon(video.thumb())
        if video.stream.live:
            d = {}
            d['url'] = self.url
            d['title'] = self.artist
            d['thumb'] = video.thumb().getvalue()
            utils.update_live(d, self.cw)


def int_(x):
    try:
        return int(x)
    except:
        return 0


@try_n(2, sleep=1)
def get_videos(url, session, type='video', only_mp4=False, audio_included=False, max_res=None, max_abr=None, cw=None):
    info = {}

    n = get_max_range(cw)

    if Downloader.get('youtube').is_channel_url(url): #5445
        reverse = utils.SD['youtube']['channel_reverse'] #5848
        tab = ''.join(url.split('/')[4:5])
        if tab == '': #5901
            url = '/'.join(url.split('/')[:4]) + '/videos'
        info = read_channel(url, n=n, cw=cw, reverse=reverse)
        info['type'] = 'channel'
        info['title'] = '[Channel] {}'.format(info['uploader'])
        if cw:
            info['urls'] = filter_range(info['urls'], cw.range)
            cw.fped = True
    elif '/playlist' in url:
        info = read_playlist(url, n=n, cw=cw)
        info['type'] = 'playlist'
        info['title'] = '[Playlist] {}'.format(info['title'])
        if cw:
            info['urls'] = filter_range(info['urls'], cw.range)
            cw.fped = True
    elif get_id(url) or url.lower().endswith('/live'):
        info['type'] = 'single'
        info['urls'] = [url]
    else:
        raise NotImplementedError(url)

    info['videos'] = [Video({'referer':url, 'type':type, 'only_mp4':only_mp4, 'audio_included':audio_included, 'max_res':max_res, 'max_abr':max_abr}) for url in info['urls']]

    return info



def read_channel(url, n, cw=None, reverse=False):
    return read_playlist(url, n, cw, reverse=reverse)


@try_n(2)
def read_playlist(url, n, cw=None, reverse=False):
    print_ = get_print(cw)

    options = {
            'extract_flat': True,
            'playlistend': n,
            'writesubtitles': True,
            }
    ydl = ytdl.YoutubeDL(options, cw=cw)
    info = ydl.extract_info(url)

    es = info['entries']
    urls = []
    for e in es:
        href = 'https://www.youtube.com/watch?v={}'.format(e['id'])
        urls.append(href)
    if reverse:
        urls = urls[::-1]
    info['urls'] = urls

    if not info.get('uploader'):
        title = info['title']
        if title.lower().endswith(' - videos'):
            title = title[:-len(' - videos')]
        info['uploader'] = title
        print_('⚠️ Fix uploader: None -> {}'.format(title))

    return info


import selector
@selector.register('youtube')
def select():
    from Qt import Qt, QDialog, QFormLayout, QLabel, QComboBox, QWidget, QVBoxLayout, QDialogButtonBox
    if utils.ui_setting.askYoutube.isChecked():
        win = QDialog(constants.mainWindow)
        win.setWindowTitle('Youtube format')
        utils.windows.append(win)
        layout = QFormLayout(win)

        youtubeCombo_type = QComboBox()
        layout.addRow('파일 형식', youtubeCombo_type)
        for i in range(utils.ui_setting.youtubeCombo_type.count()):
            youtubeCombo_type.addItem(utils.ui_setting.youtubeCombo_type.itemText(i))
            youtubeCombo_type.setItemIcon(i, utils.ui_setting.youtubeCombo_type.itemIcon(i))
        youtubeCombo_type.setCurrentIndex(utils.ui_setting.youtubeCombo_type.currentIndex())

        youtubeLabel_res = QLabel('해상도')
        youtubeCombo_res = QComboBox()
        for i in range(utils.ui_setting.youtubeCombo_res.count()):
            youtubeCombo_res.addItem(utils.ui_setting.youtubeCombo_res.itemText(i))
        youtubeCombo_res.setCurrentIndex(utils.ui_setting.youtubeCombo_res.currentIndex())

        youtubeLabel_abr = QLabel('음질')
        youtubeCombo_abr = QComboBox()
        for i in range(utils.ui_setting.youtubeCombo_abr.count()):
            youtubeCombo_abr.addItem(utils.ui_setting.youtubeCombo_abr.itemText(i))
        youtubeCombo_abr.setCurrentIndex(utils.ui_setting.youtubeCombo_abr.currentIndex())

        aa = QWidget()
        a = QVBoxLayout(aa)
        a.setContentsMargins(0,0,0,0)
        a.addWidget(youtubeLabel_res)
        a.addWidget(youtubeLabel_abr)
        bb = QWidget()
        b = QVBoxLayout(bb)
        b.setContentsMargins(0,0,0,0)
        b.addWidget(youtubeCombo_res)
        b.addWidget(youtubeCombo_abr)
        layout.addRow(aa, bb)

        def currentIndexChanged(index):
            text_type = compatstr(youtubeCombo_type.currentText())
            print(text_type)
            if tr_('동영상') in text_type:
                youtubeLabel_abr.hide()
                youtubeCombo_abr.hide()
                youtubeLabel_res.show()
                youtubeCombo_res.show()
            elif tr_('음원') in text_type:
                youtubeLabel_res.hide()
                youtubeCombo_res.hide()
                youtubeLabel_abr.show()
                youtubeCombo_abr.show()
        youtubeCombo_type.currentIndexChanged.connect(currentIndexChanged)
        youtubeCombo_type.currentIndexChanged.emit(youtubeCombo_type.currentIndex())

        buttonBox = QDialogButtonBox()
        layout.addWidget(buttonBox)
        buttonBox.setOrientation(Qt.Horizontal)
        buttonBox.setStandardButtons(QDialogButtonBox.Cancel | QDialogButtonBox.Ok)
        buttonBox.accepted.connect(win.accept)
        buttonBox.rejected.connect(win.reject)

        tr(win)
        win.setWindowOpacity(constants.opacity_max)
        try:
            res = win.exec()
            if not res:
                return selector.Cancel
            utils.windows.remove(win)
            format = {}
            format['format'] = compatstr(youtubeCombo_type.currentText()).lower().split()[0]
            format['res'] = get_resolution(compatstr(youtubeCombo_res.currentText()))
            format['abr'] = get_abr(compatstr(youtubeCombo_abr.currentText()))
        finally:
            win.deleteLater()
        return format


@selector.options('youtube')
def options(urls):
    return [
        {'text': 'MP4 (동영상)', 'format': 'mp4', 'icon': 'movie'},
        {'text': 'MP3 (음원)', 'format': 'mp3', 'icon': 'music'},
        ]

@selector.default_option('youtube')
def default_option():
    return compatstr(utils.ui_setting.youtubeCombo_type.currentText()).lower().split()[0]


def get_streamer_name(url):
    if url.endswith('/live'):
        url = url[:-len('/live')]
    ydl = ytdl.YoutubeDL({'playlistend': 0}, type='youtube')
    info = ydl.extract_info(url)
    return info['channel']


class Live_youtube(utils.Live):
    type = 'youtube'

    @classmethod
    def is_live(cls, url):
        return ''.join(url.split('/')[3:4]).startswith('@')

    @classmethod
    def fix_url(cls, url):
        cn = url.split('/')[3].split('?')[0].split('#')[0]
        return f'https://youtube.com/{cn}/live'

    @classmethod
    def check_live(cls, url, info=None):
        if info is not None:
            try:
                info['title'] = get_streamer_name(url)
            except Exception as e:
                utils.log(print_error(e))
        ydl = ytdl.YoutubeDL(type='youtube')
        try:
            _ = ydl.extract_info(url)
            return _.get('live_status') == 'is_live'
        except Exception as e:
            print(e)
            return False



File: src/extractor/pixiv_downloader.py

import downloader
from utils import Downloader, urljoin, clean_title, LazyUrl, get_ext, get_print, try_n, compatstr, get_max_range, check_alive, query_url, Soup, limits
import ffmpeg
import utils
import os
import ree as re
import errors
from translator import tr_
from error_printer import print_error
from urllib.parse import quote, unquote
import constants
from datetime import datetime
import requests
from timee import sleep
from collections import deque
from locker import lock
import threading
import clf2
from PIL import Image as Image_
##import asyncio
LIMIT = 48
for header in ['pixiv_illust', 'pixiv_bmk', 'pixiv_search', 'pixiv_following', 'pixiv_following_r18']:
    if header not in constants.available_extra:
        constants.available_extra.append(header)
utils.TOKENS['pixiv'] = ['id', 'page', 'artist', 'artistid', 'title', 'date']


class LoginRequired(errors.LoginRequired):
    def __init__(self, *args):
        super().__init__(*args, method='browser', url='https://accounts.pixiv.net/login', w=560, h=920)



class Downloader_pixiv(Downloader):
    type = 'pixiv'
    MAX_CORE = 4
    MAX_PARALLEL = 2
    keep_date = True
    STEP = 4, 16
    URLS = ['pixiv.me', 'pixiv.net']
    ACCEPT_COOKIES = [r'(.*\.)?pixiv\.(com|co|net|me)']

    def init(self):
        setattr(self.cw, 'sid?', None)
        res = clf2.solve(self.url, cw=self.cw)
        self.session = res['session'] #5105

        soup = Soup(res['html'])

        if soup.find('a', href=lambda h: h and '/login.php' in h):
            def f(html, browser=None):
                soup = Soup(html)
                for div in soup.findAll('div'):
                    if div.get('data-page-name') == 'LoginPage':
                        browser.show()
                        return False
                browser.hide()
                return True
            try:
                res = clf2.solve('https://accounts.pixiv.net/login', session=self.session, cw=self.cw, f=f, delay=3, w=560, h=920, timeout=120)
            except clf2.Timeout:
                raise LoginRequired()
            res = clf2.solve(self.url, session=self.session, cw=self.cw)
            soup = Soup(res['html'])

        err = soup.find('p', class_='error-message')
        if err: #5223
            raise errors.Invalid(f'{err.text.strip()}: {self.url}')

    @classmethod
    def fix_url(cls, url):
        rt = utils.query_url(url).get('return_to')
        if rt:
            url = urljoin(url, rt[0])

        if '/search_user.php?' in url:
            url = f'https://pixiv.me/{utils.query_url(url).get("nick")[0]}'
        if url.startswith('illust_'):
            url = f'https://www.pixiv.net/en/artworks/{url[len("illust_"):]}'
        elif url.startswith('bmk_'):
            url = f'https://www.pixiv.net/en/users/{url[len("bmk_"):]}/bookmarks/artworks'
        elif url.startswith('search_'):
            _ = quote(url[len('search_'):].replace('+', ' '))
            url = f'https://www.pixiv.net/en/tags/{_}/artworks'
        elif url.startswith('following_r18_'):
            url = 'https://www.pixiv.net/bookmark_new_illust_r18.php'
        elif url.startswith('following_'):
            url = 'https://www.pixiv.net/bookmark_new_illust.php'
        elif not re.find(r'^https?://', url) and '.' not in url:
            url = f'https://www.pixiv.net/en/users/{url}'

        #3474
        url = re.sub(r'(users/[0-9]+)/artworks$', r'\1', url)

        url = re.sub(r'[?&]p=[0-9]+$', '', url)

        if '://' not in url: #6082
            url = 'https://' + url
        return url.strip('/')

    @classmethod
    def key_id(cls, url):
        return url.replace('://www.', '://').replace('/en/', '/').replace('http://', 'https://').lower()

    def read(self):
##        loop = asyncio.new_event_loop()
##        asyncio.set_event_loop(loop)
        try:
            info = get_info(self.url, self.session, self.cw)
            self.artist = info.get('artist') #4897
            for img in info['imgs']:
                if isinstance(img, str): # local
                    self.urls.append(img)
                    continue
                self.urls.append(img.url)
            self.title = clean_title(info['title'])
        finally:
##            loop.close()
            pass


class PixivAPIError(LoginRequired): pass
class HTTPError(Exception): pass


class PixivAPI:

    def __init__(self, session, cw):
        self.session = session
        hdr = {
            'Accept': 'application/json',
            'Accept-Encoding': 'gzip, deflate', #6588
            'Accept-Language': 'en-US,en;q=0.9,ko-KR;q=0.8,ko;q=0.7,ja;q=0.6',
            'Cache-Control': 'no-cache',
            'Pragma': 'no-cache',
            'Referer': 'https://www.pixiv.net/',
            'X-User-Id': my_id(session, cw),
            }
        self.session.headers.update(hdr)

    def illust_id(self, url):
        return re.find('/artworks/([0-9]+)', url) or re.find('[?&]illust_id=([0-9]+)', url)

    def user_id(self, url):
        return re.find('/users/([0-9]+)', url) or re.find('[?&]id=([0-9]+)', url)

    @try_n(8, sleep=5)
    @limits(1.5) #3355, #5105
    def call(self, url):
        #print('call:', url)
        url = urljoin('https://www.pixiv.net/ajax/', url)
        e_ = None
        try:
            info = downloader.read_json(url, session=self.session)
        except requests.exceptions.HTTPError as e:
            code = e.response.status_code
            if code in (403, 404):
                e_ = HTTPError(f'{code} Client Error')
            else:
                raise e
        if e_:
            raise e_
        err = info['error']
        if err:
            raise PixivAPIError(info.get('message'))
        return info['body']

    def illust(self, id_):
        return self.call(f'illust/{id_}')

    def pages(self, id_):
        return self.call(f'illust/{id_}/pages')

    def ugoira_meta(self, id_):
        return self.call(f'illust/{id_}/ugoira_meta')

    def profile(self, id_):
        return self.call(f'user/{id_}/profile/all')

    def top(self, id_):
        return self.call(f'user/{id_}/profile/top')

    def bookmarks(self, id_, offset=0, limit=None, rest='show'):
        if limit is None:
            limit = LIMIT
        return self.call(f'user/{id_}/illusts/bookmarks?tag=&offset={offset}&limit={limit}&rest={rest}')

    def search(self, q, order='date_d', mode='all', p=1, s_mode='s_tag_full', type_='all', scd=None, ecd=None, wlt=None, wgt=None, hlt=None, hgt=None, blt=None, bgt=None, ratio=None, tool=None):
        url = f'search/artworks/{quote(q)}?word={quote(q)}&order={order}&mode={mode}&p={p}&s_mode={s_mode}&type={type_}'
        if scd:
            url += f'&scd={scd}'
        if ecd:
            url += f'&ecd={ecd}'
        if wlt:
            url += f'&wlt={wlt}'
        if wgt:
            url += f'&wgt={wgt}'
        if hlt:
            url += f'&hlt={hlt}'
        if hgt:
            url += f'&hgt={hgt}'
        if blt:
            url += f'&blt={blt}'
        if bgt:
            url += f'&bgt={bgt}'
        if ratio:
            url += f'&ratio={ratio}'
        if tool:
            url += f'&tool={tool}'
        return self.call(url)

    def following(self, p, r18=False): #4077
        mode = 'r18' if r18 else 'all'
        url = f'follow_latest/illust?p={p}&mode={mode}'
        return self.call(url)



class Image:
    local = False
    def __init__(self, url, referer, id_, p, info, cw, ugoira=None):
        self._url = url
        self.id_ = id_
        self.p = p
        self.artist = info['artist']
        self.artistid = info['artist_id'] #3636
        self.title = info['raw_title']
        self.utime = info['create_date']
        self.cw = cw
        self.ugoira = ugoira
        self.url = LazyUrl(referer, self.get, self, pp=self.pp, detect_local=not ugoira)

    def get(self, referer):
        d = {
            'id': self.id_,
            'page': self.p,
            'artist': clean_title(self.artist, allow_dot=True),
            'artistid': self.artistid,
            'title': clean_title(self.title, allow_dot=True), #6433, #6592
            'date': self.utime,
            }
        self.filename = utils.format('pixiv', d, get_ext(self._url))
        if self.ugoira and self.ugoira['ext']: #3355
            filename_local = os.path.join(self.cw.dir, self.filename)
            filename_local = f'{os.path.splitext(filename_local)[0]}{self.ugoira["ext"]}'
            if os.path.abspath(filename_local) in self.cw.names_old or os.path.exists(filename_local): #4534
                self.filename = os.path.basename(filename_local)
                self.local = True
        return self._url

    def pp(self, filename):
        if self.ugoira and self.ugoira['ext'] and not self.local:
            if utils.ui_setting:
                dither = utils.ui_setting.checkDither.isChecked()
                quality = utils.ui_setting.ugoira_quality.value()
            else:
                dither = True
                quality = 90
            filename_new = f'{os.path.splitext(filename)[0]}{self.ugoira["ext"]}'
            ffmpeg.gif(filename, filename_new, self.ugoira['delay'], dither=dither, quality=quality, cw=self.cw)
            utils.removeDirList.append((filename, False))
            return filename_new


def pretty_tag(tag):
    return tag.replace(' ', '').lower()


@lock
def tags_matched(tags_illust, tags_add, cw=None):
    print_ = get_print(cw)

    cache = cw.get_extra('pixiv_tag_cache') if cw else None
    init = True
    if cache is not None:
        init = False
        tags = set(cache['tags'])
        tags_ex = set(cache['tags_ex'])
    else:
        if utils.ui_setting and utils.ui_setting.groupBox_tag.isChecked():
            tags_ = [compatstr(utils.ui_setting.tagList.item(i).text()) for i in range(utils.ui_setting.tagList.count())]
        else:
            tags_ = []
        tags = set()
        tags_ex = set()
        for tag in tags_:
            tag = pretty_tag(tag)
            if tag.startswith('-'):
                tags_ex.add(tag[1:].strip())
            else:
                tags.add(tag)

    if init:
        if cw:
            cache = {}
            cache['tags'] = list(tags)
            cache['tags_ex'] = list(tags_ex)
            cw.set_extra('pixiv_tag_cache', cache)
        print_(f'tags: [{", ".join(tags)}]')
        print_(f'tags_ex: [{", ".join(tags_ex)}]')

    if tags_add:
        tags.update((pretty_tag(tag) for tag in tags_add))
        if init:
            print_(f'tags_add: {tags_add}')

    tags_illust = set(pretty_tag(tag) for tag in tags_illust)
    return (not tags or tags & tags_illust) and tags_ex.isdisjoint(tags_illust)


def get_info(url, session, cw=None, depth=0, tags_add=None):
    print_ = get_print(cw)
    api = PixivAPI(session, cw)
    info = {}
    imgs = []

    ugoira_ext = [None, '.gif', '.webp', '.png'][utils.ui_setting.ugoira_convert.currentIndex()] if utils.ui_setting else None

    max_pid = get_max_range(cw)

    if api.illust_id(url): # Single post
        id_ = api.illust_id(url)
        data = api.illust(id_)
        login = 'noLoginData' not in data
        if not login:#
            raise LoginRequired()
        if data['xRestrict'] and not login:
            raise LoginRequired('R-18')
        info['artist'] = data['userName']
        info['artist_id'] = data['userId']
        info['raw_title'] = data['illustTitle']
        info['title'] = f'{info["raw_title"]} (pixiv_illust_{id_})'
        info['create_date'] = parse_time(data['createDate'])
        tags_illust = set(tag['tag'] for tag in data['tags']['tags'])

        if tags_matched(tags_illust, tags_add, cw):
            if data['illustType'] == 2: # ugoira
                data = api.ugoira_meta(id_)
                ugoira = {
                    'ext': ugoira_ext,
                    'delay': [frame['delay'] for frame in data['frames']],
                    }
                img = Image(data['originalSrc'], url, id_, 0, info, cw, ugoira=ugoira)
                imgs.append(img)
            else:
                data = api.pages(id_)
                for img in data:
                    img = Image(img['urls']['original'], url, id_, len(imgs), info, cw)
                    imgs.append(img)
        else:
            print('tags mismatched')
    elif '/bookmarks/' in url or 'bookmark.php' in url: # User bookmarks
        id_ = api.user_id(url)
        if id_ is None: #
            id_ = my_id(session, cw)
        if id_ == my_id(session, cw):
            rests = ['show', 'hide']
        else:
            rests = ['show']
        process_user(id_, info, api)
        info['title'] = f'{info["artist"]} (pixiv_bmk_{info["artist_id"]})'
        ids = []
        ids_set = set()
        for rest in rests:
            offset = 0
            while len(ids) < max_pid:
                data = api.bookmarks(id_, offset, rest=rest)
                c = 0
                for id in [work['id'] for work in data['works']]:
                    if id in ids_set:
                        continue
                    ids_set.add(id)
                    ids.append(id)
                    c += 1
                if not c:
                    break
                offset += LIMIT
                if depth == 0:
                    check_alive(cw)
        process_ids(ids, info, imgs, session, cw, depth)
    elif '/tags/' in url or 'search.php' in url: # Search
        q = unquote(re.find(r'/tags/([^/]+)', url) or re.find('[?&]word=([^&]*)', url, err='no tags'))
        info['title'] = f'{q} (pixiv_search_{q.replace(" ", "+")})'
        qs = query_url(url)
        order = qs.get('order', ['date_d'])[0]
        mode = qs.get('mode', ['all'])[0]
        s_mode = qs.get('s_mode', ['s_tag_full'])[0]
        scd = qs.get('scd', [None])[0]
        ecd = qs.get('ecd', [None])[0]
        type_ = qs.get('type', ['all'])[0]
        wlt = qs.get('wlt', [None])[0]
        wgt = qs.get('wgt', [None])[0]
        hlt = qs.get('hlt', [None])[0]
        hgt = qs.get('hgt', [None])[0]
        blt = qs.get('blt', [None])[0]
        bgt = qs.get('bgt', [None])[0]
        ratio = qs.get('ratio', [None])[0]
        tool = qs.get('tool', [None])[0]
        logs = [
            f'order: {order}',
            f'mode: {mode}',
            f's_mode: {s_mode}',
            f'scd / ecd: {scd} / {ecd}',
            f'type: {type_}',
            f'wlt /  wgt: {wlt} / {wgt}',
            f'hlt / hgt: {hlt} / {hgt}',
            f'blt / bgt: {blt} / {bgt}',
            f'ratio: {ratio}',
            f'tool: {tool}',
                ]
        print_('\n'.join(logs))
        ids = []
        ids_set = set()
        p = 1
        while len(ids) < max_pid:
            data = api.search(q, order, mode, p=p, s_mode=s_mode, scd=scd, ecd=ecd, type_=type_, wlt=wlt, wgt=wgt, hlt=hlt, hgt=hgt, blt=blt, bgt=bgt, ratio=ratio, tool=tool)
            c = 0
            for id in [illust['id'] for illust in data['illustManga']['data'] if 'id' in illust]:
                if id in ids_set:
                    continue
                ids_set.add(id)
                ids.append(id)
                c += 1
            if not c:
                break
            p += 1
        process_ids(ids, info, imgs, session, cw, depth)
    elif 'bookmark_new_illust.php' in url or 'bookmark_new_illust_r18.php' in url or re.search(r'/users/[0-9]+/following', url): # Newest works: Following
        r18 = 'bookmark_new_illust_r18.php' in url
        id_ = my_id(session, cw)
        process_user(id_, info, api)
        info['title'] = f'{info["artist"]} (pixiv_following_{"r18_" if r18 else ""}{info["artist_id"]})'
        ids = []
        ids_set = set()
        p = 1
        while len(ids) < max_pid:
            data = api.following(p, r18=r18)
            c = 0
            for id in data['page']['ids']:
                if id in ids_set:
                    continue
                ids_set.add(id)
                ids.append(id)
                c += 1
            if not c:
                break
            p += 1
        process_ids(ids, info, imgs, session, cw, depth)
    elif api.user_id(url): # User illusts
        m = re.search(r'/users/[0-9]+/([\w]+)/?([^\?#/]*)', url)
        type_ = {'illustrations': 'illusts', 'manga': 'manga'}.get(m and m.groups()[0])
        if type_:
            types = [type_]
        else:
            types = ['illusts', 'manga']
        if m:
            tag = unquote(m.groups()[1]) or None
        else:
            tag = None
        print_(f'types: {types}, tag: {tag}')

        id_ = api.user_id(url)
        process_user(id_, info, api)
        data = api.profile(id_)
        info['title'] = f'{info["artist"]} (pixiv_{info["artist_id"]})'

        ids = []
        for type_ in types:
            illusts = data[type_]
            if not illusts:
                continue
            ids += list(illusts.keys())
        ids = sorted(ids, key=int, reverse=True)
        print_(f'ids: {len(ids)}')
        if not ids:
            raise Exception('no imgs')
        process_ids(ids, info, imgs, session, cw, depth, tags_add=[tag] if tag else None)
    else:
        raise NotImplementedError()
    info['imgs'] = imgs[:max_pid]

    return info


def parse_time(ds):
    ds, z = ds[:-6], ds[-6:]
    dt = int(z[:3]) * 3600 + int(z[4:]) * 60
    time = datetime.strptime(ds.replace('  ', ' '), '%Y-%m-%dT%H:%M:%S')
    time = (time-datetime(1970,1,1)).total_seconds()
    return time - dt


@try_n(4, sleep=.5) #5469
def my_id(session, cw):
    print_ = get_print(cw)
    sid = session.cookies.get('PHPSESSID', domain='.pixiv.net', path='/')
    if not sid:
        raise LoginRequired()
    if cw is not None:
        _ = getattr(cw, 'sid?', None)
        if _ is None:
            setattr(cw, 'sid?', sid)
            print_(f'sid: {sid}')
    userid = re.find(r'^([0-9]+)', sid)
    if userid is None:
        raise LoginRequired()
    return userid


def process_user(id_, info, api):
    info['artist_id'] = id_
    data_user = api.top(id_)
    info['artist'] = data_user['extraData']['meta']['ogp']['title']


def process_ids(ids, info, imgs, session, cw, depth=0, tags_add=None):
    print_ = get_print(cw)
    max_pid = get_max_range(cw)

    names = cw.names_old
    table = {}
    for name in names:
        id = re.find(r'([0-9]+)_p[0-9]+.*\.(jpg|jpeg|png|apng|bmp|webp|gif)$', os.path.basename(name)) #5541
        if id is None:
            continue
        ext = os.path.splitext(name)[1]
        if ext.lower() in ['.gif', '.webp']: #5541
            try:
                img = Image_.open(name)
                n_frames = getattr(img, 'n_frames', 1)
            except Exception as e:
                print_(print_error(e))
                n_frames = 1
            if n_frames > 1:
                print_(f'ugoira: {name}')
                continue
        id = id[0]
        if id in table:
            table[id].append(name)
        else:
            table[id] = [name]

    c_old = 0

    class Thread(threading.Thread):
        alive = True
        rem = 0

        def __init__(self, queue):
            super().__init__(daemon=True)
            self.queue = queue

        @classmethod
        @lock
        def add_rem(cls, x):
            cls.rem += x

        def run(self):
            nonlocal c_old
            while self.alive:
                try:
                    id_, res, i = self.queue.popleft()
                except:
                    sleep(.1)
                    continue
                try:
                    names = table.get(str(id_))
                    if names is not None:
                        res[i] = utils.natural_sort(names)
                        c_old += 1
                    else:
                        info_illust = get_info(f'https://www.pixiv.net/en/artworks/{id_}', session, cw, depth=depth+1, tags_add=tags_add)
                        res[i] = info_illust['imgs']
                except Exception as e:
                    if depth == 0 and (e.args and e.args[0] == '不明なエラーが発生しました' or isinstance(e, errors.LoginRequired)): # logout during extraction
                        res[i] = e
                    print_(f'process_ids error (id: {id_}, d:{depth}):\n{print_error(e)}')
                finally:
                    Thread.add_rem(-1)
    queue = deque()
    n, step = Downloader_pixiv.STEP
    print_(f'{n} / {step}')
    ts = []
    for i in range(n):
        t = Thread(queue)
        t.start()
        ts.append(t)
    for i in range(0, len(ids), step):
        res = [[]]*step
        for j, id_illust in enumerate(ids[i:i+step]):
            queue.append((id_illust, res, j))
            Thread.add_rem(1)
        while Thread.rem:
            sleep(.01, cw)
        for imgs_ in res:
            if isinstance(imgs_, Exception):
                raise imgs_
            imgs += imgs_
        s = f'{tr_("읽는 중...")} {info["title"]} - {len(imgs)}'
        if cw:
            cw.setTitle(s)
        else:
            print(s)
        if len(imgs) >= max_pid:
            break
        if depth == 0:
            check_alive(cw)
    for t in ts:
        t.alive = False

    print_(f'c_old: {c_old}')



File: src/extractor/kakaotv_downloader.py

import downloader
import ytdl
from utils import Downloader, try_n, LazyUrl, get_ext, format_filename
from io import BytesIO as IO



class Downloader_kakaotv(Downloader):
    type = 'kakaotv'
    URLS = ['tv.kakao']
    single = True
    display_name = 'KakaoTV'
    ACCEPT_COOKIES = [r'(.*\.)?kakao\.com']

    @classmethod
    def fix_url(cls, url):
        url = url.replace('.kakao.com/m/', '.kakao.com/')
        return url.split('?')[0].strip('/')

    def read(self):
        video = Video(self.url, cw=self.cw)
        video.url()#

        self.urls.append(video.url)
        self.setIcon(video.thumb)

        self.enableSegment()

        self.title = video.title



class Video:
    _url = None

    def __init__(self, url, cw=None):
        self.url = LazyUrl(url, self.get, self)
        self.cw = cw

    @try_n(2)
    def get(self,  url):
        if self._url:
            return self._url

        ydl = ytdl.YoutubeDL(cw=self.cw)
        info = ydl.extract_info(url)
        fs = [f for f in info['formats'] if f['ext'] == 'mp4']
        f = sorted(fs, key=lambda f: f['height'])[-1]
        self._url = f['url']

        self.thumb_url = info['thumbnails'][0]['url']
        self.thumb = IO()
        downloader.download(self.thumb_url, buffer=self.thumb)
        self.title = info['title']
        ext = get_ext(self._url)
        self.filename = format_filename(self.title, info['id'], ext)
        return self._url



File: src/extractor/torrent_downloader.py

from utils import Downloader, clean_title, lock, json
import constants, os, downloader
from size import Size
from timee import sleep
from translator import tr_
import utils
import filesize as fs
from datetime import datetime
import errors
import ips
import order
from cacher import Cache
torrent = None
TIMEOUT = 1800
CACHE_INFO = True


def isInfoHash(s):
    if len(s) != 40:
        return False
    try:
        bytes.fromhex(s)
        return True
    except:
        return False



class Downloader_torrent(Downloader):
    type = 'torrent'
    URLS = [r'regex:^magnet:', r'regex:\.torrent$', isInfoHash]
    single = True
    update_filesize = False
    _info = None
    _name = None
    _filesize_prev = 0
    _upload_prev = 0
    _state = None
    _h = None
    _dn = None
    MAX_PARALLEL = 16
    MAX_CORE = 0
    skip_convert_imgs = True
    _filesize_init = False
    _max_speed = None
    _anon = False
    _proxy = '', '', 0, '', ''
    _seeding = False
    _virgin = True
    STOP_READING = False
    PRIORITY = -1

    @classmethod
    def fix_url(cls, url):
        if isInfoHash(url):
            url = f'magnet:?xt=urn:btih:{url}'
        return url

    @classmethod
    def set_max_speed(cls, speed):
        cls._max_speed = speed
        cls.updateSettings()

    @classmethod
    def set_anon(cls, flag):
        cls._anon = flag
        cls.updateSettings()

    @classmethod
    def set_proxy(cls, protocol, host, port, username, password):
        cls._proxy = protocol, host, port, username, password
        cls.updateSettings()

    @classmethod
    @lock
    def updateSettings(cls):
        if torrent is None:
            print('torrent is None')
            return
        torrent.set_max_speed(cls._max_speed)
        torrent.set_anon(cls._anon)
        torrent.set_proxy(*cls._proxy)

    @classmethod
    def _import_torrent(cls):
        global torrent
        if torrent is None:
            import torrent

    @lock
    def __init(self):
        self._import_torrent()
        Downloader_torrent.updateSettings()

    @classmethod
    def key_id(cls, url):
        if torrent is None:
            #print('torrent is None')
            return url
        id_, e = torrent.key_id(url)
        if e:
            print(e)
        return id_

    @property
    def name(self):
        if self._name is None:
            self._name = clean_title(self._info.name())
        return self._name

    @classmethod
    def get_dn(cls, url):
        if not url:
            return
        if url.startswith('magnet:'):
            qs = utils.query_url(url)
            if 'dn' in qs:
                return utils.html_unescape(qs['dn'][0])

    def read(self):
        cw = self.cw
        self.cw.pbar.hide()
        self.__init()
        if cw:
            cw._torrent_s = None
        self._dn = self.get_dn(cw.gal_num)
        info = getattr(cw, 'info?', None)
        if info is not None:
            self.print_('cached info')
            self._info = info
        if self._info is None:
            if not (self.url.startswith('http') or self.url.startswith('magnet:')) and not os.path.exists(self.url):
                sr = cw.serial_retry
                if sr is not None:
                    self.url = json.loads(sr)['url'] or self.url
            try:
                self._info = torrent.get_info(self.url, cw, timeout=TIMEOUT, callback=self.callback)
                if CACHE_INFO:
                    setattr(cw, 'info?', self._info)
            except:
                self.update_pause()
                if not cw.paused:
                    raise errors.Invalid(f'Faild to read metadata: {self.url}', fail=True)
        if self._info is None:
            cw.paused = True
        if cw.paused:
            return
        hash_ = self._info.hash.hex()
        self.print_(f'v2: {self._info.v2}')
        self.print_(f'Hash: {hash_}')
        if not self._info.v2:
            self.url = f'magnet:?xt=urn:btih:{hash_}'#
        date = datetime.fromtimestamp(self._info.creation_date())
        date = date.strftime('%y-%m-%d %H:%M:%S')
        self.print_(f'Created on: {date}')
        self.print_(f'Total size: {fs.size(self._info.total_size())}')
        self.print_(f'Pieces: {self._info.num_pieces()} x {fs.size(self._info.piece_length())}')
        self.print_(f'Creator: {self._info.creator()}')
        self.print_(f'Comment: {self._info.comment()}')
        cw.setTotalFileSize(self._info.total_size())

        cw.imgs.clear()
        cw.dones.clear()

        self.urls = [self.url]
        self.title = self.name
        self.update_files()

        if not self.single and not os.path.isdir(self.dir): #4698
            downloader.makedir_event(self.dir, cw)

        cw.pbar.show()

    def update_files(self):
        cw = self.cw
        files = torrent.get_files(self._info, cw=cw)
        if not files:
            raise Exception('No files')
        cw.single = self.single = len(files) <= 1
        index = [0]*len(files)
        filesize = []
        for i, file in enumerate(files):
            filename = os.path.join(self.dir, file.path)
            cw.imgs.append(filename)
            index[file.index] = i
            filesize.append(file.size)
        self._torrent_index = index
        self._torrent_filesize = filesize

    def update_pause(self):
        cw = self.cw
        if cw.pause_lock:
            if self._seeding:
                cw.pause_lock = False
                return
            cw.pause_data = {
                'type': self.type,
                'url': self.url,
                }
            cw.paused = True
            cw.pause_lock = False
            self.update_tools_buttons()

    def start_(self):
        cw = self.cw
        cw.pbar.setFormat('%p%')
        cw.setColor('reading')
        cw.downloader_pausable = True
        self._seeding = False
        pr = cw.get_extra('pr')
        if cw.paused:
            #data = cw.pause_data
            cw.paused = False
            cw.pause_lock = False
            self.update_tools_buttons()
        try:
            self.read()
            if self.status == 'stop':
                self.stop()
                return True
            if cw.paused:
                pass
            else:
                cw.dir = self.dir
                cw.urls[:] = self.urls
                cw.clearPieces()
                self.size = Size()
                self.size_upload = Size()
                cw.pbar.setMaximum(self._info.total_size())
                cw.setColor('reading')
                if pr is None and utils.ui_setting.torrentSelectFiles.isChecked():
                    from utils import QApplication, QStyle, QIcon
                    cache_icon = Cache(1024)
                    files = torrent.get_files(self._info)
                    icon_size = QApplication.style().pixelMetric(QStyle.PM_ListViewIconSize)
                    def _getIcon(name):
                        ext = os.path.splitext(name)[1]
                        key_icon = icon_size, ext.lower()
                        icon = cache_icon.get(key_icon)
                        if icon is None:
                            pixmap = utils.image_reader.getFilePixmap(name, size=icon_size, pad=0)
                            icon = QIcon()
                            icon.addPixmap(pixmap)
                            cache_icon.set(key_icon, icon)
                        return icon
                    done = False
                    res = None
                    def f():
                        nonlocal done, res
                        try:
                            while True:
                                res = order.getOrder([[True, file.path] for file in files], utils.ui.listWidget, self.title, tr_('파일을 고르세요:'), True, size=(600, 600), icon=_getIcon, move=False)
                                if res is None or any(item[0] for item in res):
                                    break
                                utils.messageBox('No files selected', self.title, icon=utils.QMessageBox.Warning, parent=utils.ui.listWidget)
                        finally:
                            done = True
                    utils.exec_queue.run(f)
                    while True:
                        sleep(1, cw)
                        if done:
                            break
                    if res:
                        pr = [None] * len(files)
                        for file, item in zip(files, res):
                            pr[file.index] = int(item[0])
                        cw.set_extra('pr', pr)
                    else:
                        raise errors.Invalid(f'Canceled: {self.url}')
                torrent.download(self._info, save_path=self.dir, callback=self.callback, cw=cw, pr=pr)
                self.update_progress(self._h)
                cw.setSpeed(0.0)
                cw.setUploadSpeed(0.0)
            if not cw.alive:
                return
            self.update_pause()
            if cw.paused:
                return True
            self.title = self.name
            if not self.single:
                cw.pbar.setMaximum(len(cw.imgs))
        finally:
            cw.clearPieces()
            try: # for Item.showFiles
                cw.set_extra('torrent_progress', torrent.get_file_progress(self._h, self._info, False))
            except Exception as e:
                cw.remove_extra('torrent_progress')
                self.print_error(e)
            self._h = None

    def _updateIcon(self):
        cw = self.cw
        n = 4
        for try_ in range(n):
            if cw.setIcon(cw.imgs[0], icon=try_==n-1):
                break
            sleep(.5)

    def update_progress(self, h):
        if self._info is None:
            return
        cw = self.cw

        if not cw.imgs: #???
            self.print_('???')
            self.update_files()

        cw.setPieces(torrent.pieces(h, self._info))

    def callback(self, h, s, alerts):
        try:
            return self._callback(h, s, alerts)
        except Exception as e:
            self.print_error(e)
            return 'abort'

    def _callback(self, h, s, alerts):
        self._h = h
        cw = self.cw

        if self._virgin:
            self._virgin = False
            try:
                ips.get('0.0.0.0')
            except Exception as e:
                self.print_error(e)

        if self._state != s.state_str:
            self._state = s.state_str
            self.print_(f'state: {s.state_str}')


        title = (self._dn or self.url) if self._info is None else self.name

        try:
            if cw.alive and cw.valid and not cw.pause_lock:
                seeding = False
                cw._torrent_s = s
                self.update_progress(h)

                filesize = s.total_done
                upload = s.total_upload
                color = 'downloading'
                if s.state_str in ('downloading', 'seeding'):
                    # init filesize
                    if not self._filesize_init:
                        self._filesize_prev = filesize
                        self._filesize_init = True
                        self.print_(f'init filesize: {fs.size(filesize)}')

                    # download
                    d_size = filesize - self._filesize_prev
                    self._filesize_prev = filesize
                    self.size += d_size
                    downloader.total_download_size_torrent += d_size
                    # upload
                    d_size = upload - self._upload_prev
                    self._upload_prev = upload
                    self.size_upload += d_size
                    downloader.total_upload_size_torrent += d_size
                if self._info is not None:
                    cw.pbar.setValue(s.progress * self._info.total_size())
                if s.state_str == 'queued':
                    color = 'reading'
                    title_ = f'{tr_("대기 중...")} {title}'
                elif s.state_str == 'checking files':
                    color = 'reading'
                    title_ = f'{tr_("파일 체크 중...")} {title}'
                    self._filesize_prev = filesize
                elif s.state_str == 'downloading':
                    title_ = f'{title}'
                    cw.setFileSize(filesize)
                    cw.setSpeed(self.size.speed)
                    cw.setUploadSpeed(self.size_upload.speed)
                elif s.state_str == 'seeding':
                    cw.setFileSize(filesize)
                    if not cw.seeding:
                        return 'abort'
                    seeding = True
                    title_ = f'{tr_("시딩...")} {title}'
                    cw.setSpeed(self.size_upload.speed)
                elif s.state_str == 'reading':
                    color = 'reading'
                    title_ = f'{tr_("읽는 중...")} {title}'
                elif s.state_str == 'finished':
                    return 'abort'
                else:
                    title_ = f'{s.state_str.capitalize()}... {title}'
                cw.setTitle(title_, update_filter=False)
                cw.setColor(color)
                self._seeding = seeding
            else:
                self.print_('abort')
                if cw:
                    cw._torrent_s = None
                return 'abort'
        finally:
            if alerts:
                if not cw.imgs: #???
                    self.print_('??? 2')
                    self.update_files()
                names = cw.names
                for alert in alerts:
                    what = alert['what']
                    if what == 'file_completed':
                        index = alert['index']
                        index = self._torrent_index[index]
                        try:
                            file = os.path.abspath(names[index])
                        except IndexError:
                            continue #???
                        cw.dones.add(file)
                        file = constants.compact(file).replace('\\', '/')
                        files = file.split('/')
                        file = ' / '.join(files[1:])
                        filesize = self._torrent_filesize[index]
                        msg = f'Completed: {file} | {fs.size(filesize)}'
                        self.print_(msg)
                        if index == 0:
                            self._updateIcon()
                    else:
                        raise NotImplementedError(what)


@utils.actions('torrent')
def actions(cw):
    if cw.type != 'torrent':
        return
    items = [item for item in cw.listWidget().selectedItems() if item.type == 'torrent']
    seeding = int(all(item._seeding for item in items)) * 2
    if not seeding:
        seeding = int(all(item._seeding is False for item in items))
        if not seeding:
            seeding = 0 if all(item._seeding is None for item in items) else None
    if seeding is None:
        mix_seeding = any(item._seeding for item in items)
        mix_no_seeding = any(item._seeding is False for item in items)
        mix_pref = any(item._seeding is None for item in items)
    else:
        mix_seeding = mix_no_seeding = mix_pref = False
    return [
        {'icon': 'list', 'text': '파일 목록', 'clicked': cw.showFiles},
        {'icon': 'peer', 'text': 'Peers', 'clicked': cw.showPeers},
        {'icon': 'tracker', 'text': '트래커 수정', 'clicked': cw.editTrackers},
        {'text':'-'},
        {'text': '시딩', 'clicked': lambda:cw.setSeedings(True), 'checkable': True, 'checked': seeding==2, 'group': 'seeding', 'mixed': mix_seeding},
        {'text': '시딩 하지 않음', 'clicked': lambda:cw.setSeedings(False), 'checkable': True, 'checked': seeding==1, 'group': 'seeding', 'mixed': mix_no_seeding},
        {'text': '설정을 따름', 'clicked': lambda:cw.setSeedings(None), 'checkable': True, 'checked': seeding==0, 'group': 'seeding', 'mixed': mix_pref},
        ]



File: src/extractor/wayback_machine_downloader.py

# coding: utf8
# title: Wayback Machine Downloader
# author: bog_4t
import downloader
import concurrent.futures
import os
import ree as re
from hashlib import md5
from ratelimit import limits, sleep_and_retry
from utils import Downloader, Session, clean_title, get_print, print_error



class Downloader_wayback_machine(Downloader):
    type = 'waybackmachine'
    URLS = ['archive.org', 'web.archive.org']
    display_name = 'Wayback Machine'
    MAX_CORE = 1

    def read(self):
        filter_ = Filter(self.url, self.cw)
        self.url = f'https://web.archive.org/cdx/search/cdx?url={filter_.url}'
        self.title = filter_.title
        self.urls.extend(get_imgs(self.url, filter_, self.dir, self.session, self.cw))
        self.title = filter_.title


class WaybackMachineAPI:
    def __init__(self, session, cw=None):
        self.session = session
        self.cw = cw
        self.params = {
            'output': 'json',
            'fl': 'timestamp,original',
            'filter': 'mimetype:text/html&filter=statuscode:200',
            'collapse': 'urlkey'
        }

    @sleep_and_retry
    @limits(1, 5)
    def call(self, url):
        for (key, value) in self.params.items():
            url += f'&{key}={value}'
        return downloader.read_json(url, session=self.session)

    def snapshots(self, url):
        data = self.call(url)
        return data[1:]


class Filter:
    domains = [
        'twitter.com'
    ]

    def __init__(self, url, cw=None):
        self.cw = cw
        self.url = re.findall(r'archive.[^/]+/(?:cdx/search/cdx\?url=|(?:web/)?(?:[^/]+/))(.+)', url.lower())[0].strip('/')
        self.base_url = self.url.split('&')[0].strip('/')
        self.md5 = md5(self.url.encode('utf8')).hexdigest()[:8]
        self.mode = self.__get_mode()
        self.title = self.__get_title()

    def __get_mode(self):
        for mode in (mode for mode, domain in enumerate(self.domains, start=1) if domain in self.url):
            return mode
        return 0

    def __get_title(self):
        def default():
            tail = f" ({md5(self.base_url.encode('utf8')).hexdigest()[:8]})"
            return clean_title(os.path.basename(self.base_url), n=-len(tail)) + tail

        def twitter():
            return '@' + re.findall('twitter.[^/]+/([^/*?]+)', self.url)[0]

        return [
            default,
            twitter
        ][self.mode]()


class Bitmap:
    bitmask = [0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80]

    def __init__(self, size=0, cw=None):
        self.cw = cw
        self.bitmap = bytearray([False] * ((size + 7) // 8))

    def set(self, index):
        self.bitmap[index // 8] |= self.bitmask[index % 8]

    def unset(self, index):
        self.bitmap[index // 8] &= ~self.bitmask[index % 8]

    def get(self, index):
        return (self.bitmap[index // 8] & (self.bitmask[index % 8])) != 0

    def save(self, path):
        with open(path, 'wb') as file:
            file.seek(0)
            file.write(self.bitmap)

    def load(self, size, path):
        with open(path, 'rb') as file:
            self.bitmap = bytearray(file.read((size + 7) // 8))
        return self

    def update(self, id_, path):
        self.set(id_)
        self.save(path)


def get_imgs(url, filter_, directory, session=Session(), cw=None):
    print_ = get_print(cw)

    if not os.path.exists(directory):
        os.makedirs(directory)

    urls_path = os.path.join(directory, '{}.urls'.format(filter_.md5))
    bitmap_path = os.path.join(directory, '{}.bitmap'.format(filter_.md5))
    count_path = os.path.join(directory, '{}.count'.format(filter_.md5))

    for path in [urls_path, bitmap_path, count_path]:
        if not os.path.exists(path):
            open(path, 'x').close()

    with open(count_path) as file:
        num_complete = (lambda x: int(x) if x else 0)(file.read())

    snapshots = WaybackMachineAPI(session, cw).snapshots(url)
    bitmap = Bitmap(cw=cw).load(len(snapshots), bitmap_path) if num_complete else Bitmap(len(snapshots), cw=cw)

    base_url = 'https://web.archive.org/web/{}im_/{}'

    def get_imgs_snapshot(id_, snapshot):

        @sleep_and_retry
        @limits(1, 5)
        def get_soup():
            try:
                return downloader.read_soup(f'https://web.archive.org/web/{snapshot[0]}id_/{snapshot[1]}')
            except Exception as exception:
                print_(print_error(exception)[0])
                return None

        def get_imgs_soup(soup):
            if not soup:
                return []

            def default():
                return [base_url.format(snapshot[0], img['src']) for img in soup.find_all('img', src=True)]

            def twitter():
                return [base_url.format(snapshot[0], img['src']) for img in soup.find_all('img', src=True) if 'twimg.com/media/' in img['src']]

            return [
                default,
                twitter
            ][filter_.mode]()

        return id_, get_imgs_soup(get_soup())

    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        futures = [executor.submit(get_imgs_snapshot, id_, snapshot) for id_, snapshot in enumerate(snapshots) if not bitmap.get(id_)]

    with open(urls_path, 'a') as urls_file:
        for future in concurrent.futures.as_completed(futures):
            id_, urls = future.result()
            urls_file.writelines([f'{url}\n' for url in urls])
            bitmap.update(id_, bitmap_path)
            num_complete += 1
            with open(count_path, 'w') as count_file:
                count_file.write(str(num_complete))
            msg = f'{filter_.title} - {num_complete}'
            cw.setTitle(msg) if cw else print_(msg)

    with open(urls_path) as file:
        urls = set()
        for url in file.readlines():
            urls.update(re.findall(r'^\S+$', url))

    os.remove(urls_path)
    os.remove(bitmap_path)
    os.remove(count_path)

    return urls



File: src/extractor/luscious_downloader.py

#coding:utf8
import downloader
import utils
from utils import Soup, Downloader, LazyUrl, urljoin, try_n, clean_title, get_max_range, json
import ree as re
import os
from translator import tr_
from io import BytesIO
import clf2
import errors
downloader.REPLACE_UA[r'\.luscious\.net'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36'


class LoginRequired(errors.LoginRequired):
    def __init__(self, *args):
        super().__init__(*args, method='browser', url='https://members.luscious.net/login/')


class Image:
    def __init__(self, item, referer):
        self.item = item
        self.id = str(item['id'])
        self.referer = referer
        self.url = LazyUrl(referer, self.get, self)

    def get(self, url):
        img = urljoin(url, self.item['url_to_original'])
        ext = os.path.splitext(img.split('?')[0])[1]
        self.filename = '{}{}'.format(self.id, ext)
        return img


class Video:
    id = None
    def __init__(self, url, title, url_thumb):
        self.url = url
        self.title = title
        ext = os.path.splitext(url.split('?')[0])[1]
        self.filename = '{}{}'.format(clean_title(title), ext)
        self.url_thumb = url_thumb
        self.thumb = BytesIO()
        downloader.download(self.url_thumb, buffer=self.thumb)



class Downloader_luscious(Downloader):
    type = 'luscious'
    URLS = ['luscious.net']
    MAX_CORE = 4
    ACCEPT_COOKIES = [r'(.*\.)?luscious\.net']

    @classmethod
    def fix_url(cls, url):
        url = url.replace('members.luscious.', 'www.luscious.')
        return url

    @classmethod
    def key_id(cls, url):
        return '/'.join(url.split('/')[3:])

    def read(self):
        def f(html, browser=None):
            soup = Soup(html)
            if soup.find('input', type='password'):
                browser.show()
                return False
            browser.hide()
            try:
                get_title(soup)
            except:
                return False
            return True
        res = clf2.solve(self.url, f=f, cw=self.cw, show=True)
        self.url = res['url']
        soup = Soup(res['html'])
        if soup.find(class_='http-error-404-page-container'):
            raise LoginRequired(get_title(soup)) #6912

        title = clean_title(get_title(soup))

        self.title = tr_('읽는 중... {}').format(title)

        if '/videos/' in self.url:
            video = get_video(self.url, soup)
            imgs = [video]
            self.setIcon(video.thumb)
        else:
            imgs = get_imgs(self.url, soup, self.cw)

        dir = utils.dir(self.type, title, self.cw)
        names = {}
        try:
            for name in os.listdir(dir):
                id = os.path.splitext(name)[0]
                names[id] = name
        except:
            pass

        for img in imgs:
            if img.id in names:
                url = os.path.join(dir, names[img.id])
            else:
                url = img.url
            self.urls.append(url)

        self.title = title#


def update(cw, title, imgs):
    s = '{} {} - {}'.format(tr_('읽는 중...'), title, len(imgs))
    if cw is not None:
        cw.setTitle(s)
    else:
        print(s)

def get_imgs(url, soup=None, cw=None):
    if soup is None:
        html = downloader.read_html(url)
        soup = Soup(html)
    title = get_title(soup)

    n = get_max_range(cw)

    imgs = []
    p = 1
    while True:
        imgs_new, has_next_page = get_imgs_p(url, p)
        if not imgs_new:
            break
        imgs += imgs_new
        update(cw, title, imgs)
        p += 1
        if len(imgs) >= n or not has_next_page:
            break
    return imgs[:n]


@try_n(4, sleep=30)
def get_imgs_p(url, p=1):
    id = re.find('/albums/[^/]+?([0-9]+)/', url+'/')
    print(url, id)
    #5699
    #url_api = 'https://api.luscious.net/graphql/nobatch/?operationName=AlbumListOwnPictures&query=+query+AlbumListOwnPictures%28%24input%3A+PictureListInput%21%29+%7B+picture+%7B+list%28input%3A+%24input%29+%7B+info+%7B+...FacetCollectionInfo+%7D+items+%7B+...PictureStandardWithoutAlbum+%7D+%7D+%7D+%7D+fragment+FacetCollectionInfo+on+FacetCollectionInfo+%7B+page+has_next_page+has_previous_page+total_items+total_pages+items_per_page+url_complete+%7D+fragment+PictureStandardWithoutAlbum+on+Picture+%7B+__typename+id+title+created+like_status+number_of_comments+number_of_favorites+status+width+height+resolution+aspect_ratio+url_to_original+url_to_video+is_animated+position+tags+%7B+category+text+url+%7D+permissions+url+thumbnails+%7B+width+height+size+url+%7D+%7D+&variables=%7B%22input%22%3A%7B%22filters%22%3A%5B%7B%22name%22%3A%22album_id%22%2C%22value%22%3A%22{}%22%7D%5D%2C%22display%22%3A%22position%22%2C%22page%22%3A{}%7D%7D'.format(id, p)
    url_api = f'https://apicdn.luscious.net/graphql/nobatch/?operationName=AlbumListOwnPictures&query=%2520query%2520AlbumListOwnPictures%28%2524input%253A%2520PictureListInput%21%29%2520%257B%2520picture%2520%257B%2520list%28input%253A%2520%2524input%29%2520%257B%2520info%2520%257B%2520...FacetCollectionInfo%2520%257D%2520items%2520%257B%2520__typename%2520id%2520title%2520description%2520created%2520like_status%2520number_of_comments%2520number_of_favorites%2520moderation_status%2520width%2520height%2520resolution%2520aspect_ratio%2520url_to_original%2520url_to_video%2520is_animated%2520position%2520tags%2520%257B%2520category%2520text%2520url%2520%257D%2520permissions%2520url%2520thumbnails%2520%257B%2520width%2520height%2520size%2520url%2520%257D%2520%257D%2520%257D%2520%257D%2520%257D%2520fragment%2520FacetCollectionInfo%2520on%2520FacetCollectionInfo%2520%257B%2520page%2520has_next_page%2520has_previous_page%2520total_items%2520total_pages%2520items_per_page%2520url_complete%2520%257D%2520&variables=%7B%22input%22%3A%7B%22filters%22%3A%5B%7B%22name%22%3A%22album_id%22%2C%22value%22%3A%22{id}%22%7D%5D%2C%22display%22%3A%22rating_all_time%22%2C%22items_per_page%22%3A50%2C%22page%22%3A{p}%7D%7D'
    data_raw = downloader.read_html(url_api, referer=url)
    data = json.loads(data_raw)
    imgs = []
    for item in data['data']['picture']['list']['items']:
        img = Image(item, url)
        imgs.append(img)

    return imgs, data['data']['picture']['list']['info']['has_next_page']


def get_video(url, soup):
    title = re.find('videos/([^/]+)', url)
    video = soup.find('video')
    url = urljoin(url, video.source.attrs['src'])
    url_thumb = urljoin(url, video['poster'])

    video = Video(url, title, url_thumb)
    return video


def get_title(soup):
    return soup.find('h1').text.strip()



File: src/extractor/hameln_downloader.py

#coding: utf8
import downloader
import os
import utils
from utils import Soup, urljoin, get_text, LazyUrl, try_n, Downloader, lazy, clean_title
import ree as re
from io import BytesIO
from translator import tr_



class Downloader_hameln(Downloader):
    type = 'hameln'
    URLS = ['syosetu.org']
    MAX_CORE = 2
    detect_removed = False
    ACCEPT_COOKIES = [r'(.*\.)?syosetu\.org']

    def init(self):
        id_ = re.find('/novel/([^/]+)', self.url)
        if id_ is not None:
            self.url = 'https://syosetu.org/novel/{}/'.format(id_)

    @lazy
    def soup(self):
        html = read_html(self.url)
        soup = Soup(html)
        return soup

    @lazy
    def info(self):
        return get_info(self.url, self.soup)

    def read(self):
        for page in get_pages(self.url, self.soup):
            text = Text(page, len(self.urls)+1)
            self.urls.append(text.url)

        self.artist = self.info['artist']
        self.title = clean_title('[{}] {}'.format(self.artist, self.info['title']), n=-len('[merged] .txt'))

    def post_processing(self):
        names = self.cw.names
        filename = os.path.join(self.dir, '[merged] {}.txt'.format(self.title))
        try:
            with utils.open(filename, 'wb') as f:
                f.write('    {}\n\n    作者：{}\n\n\n'.format(self.info['title'], self.artist).encode('utf8'))
                if self.info['novel_ex']:
                    f.write(self.info['novel_ex'].encode('utf8'))
                for i, file in enumerate(names):
                    self.cw.pbar.setFormat('[%v/%m]  {} [{}/{}]'.format(tr_('병합...'), i, len(names)))
                    with open(file, 'rb') as f_:
                        text = f_.read()
                    f.write(b'\n\n\n\n')
                    f.write(text)
        finally:
            self.cw.pbar.setFormat('[%v/%m]')


class Text:
    def __init__(self, page, p):
        self.page = page
        self.url = LazyUrl(page.url, self.get, self)
        self.filename = clean_title('[{:04}] {}'.format(p, page.title), n=-4) + '.txt'

    def get(self, url):
        text = read_page(self.page)
        f = BytesIO()
        f.write(text.encode('utf8'))
        f.seek(0)
        return f


class Page:
    def __init__(self, title, url):
        self.title = clean_title(title)
        self.url = url



def read_html(url):
    return downloader.read_html(url, cookies={'over18': 'off'}, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'})


def get_sss(soup):
    sss = [ss for ss in soup.findAll('div', class_='ss') if ss.attrs.get('id')!='fmenu']
    return sss


def get_pages(url, soup=None):
    if soup is None:
        html = read_html(url)
        soup = Soup(html)

    sss = get_sss(soup)
    list = sss[-1]

    pages = []
    for tr in list.findAll('tr'):
        a = tr.find('a')
        if a is None:
            continue
        text =a.text.strip()
        href = urljoin(url, a.attrs['href'])
        page = Page(text, href)
        pages.append(page)

    return pages


@try_n(22, sleep=30)
def read_page(page):
    html = read_html(page.url)
    soup = Soup(html)

    text_top = get_text(soup.find('div', id='maegaki'))
    print(text_top.count('\n'))
    text_mid = get_text(soup.find('div', id='honbun'))
    text_bot = get_text(soup.find('div', id='atogaki'))

    texts = [text for text in (text_top, text_mid, text_bot) if text]

    story = '''

────────────────────────────────

'''.join(texts)

    text = '''────────────────────────────────

  ◆  {}

────────────────────────────────


{}'''.format(page.title, story)

    return text


def get_info(url, soup=None):
    if soup is None:
        html = read_html(url)
        soup = Soup(html)

    info = {}
    info['artist'] = soup.find('span', {'itemprop':'author'}).text.strip()
    info['title'] = soup.find('span', {'itemprop':'name'}).text.strip()
    sss = get_sss(soup)
    info['novel_ex'] = get_text(sss[-2])
    return info



File: src/extractor/jmana_downloader.py

import downloader
from utils import Soup, urljoin, Downloader, fix_title, Session, get_print, LazyUrl, clean_title, get_imgs_already, check_alive, try_n, clean_url
import ree as re
from timee import sleep
from translator import tr_
import page_selector
import bs4
import clf2
PATTERN = r'jmana[0-9]*.*/(comic_list_title|book)\?book'
PATTERN_ALL = r'jmana[0-9]*.*/((comic_list_title|book|bookdetail)\?book|book_by_title\?title)' #6157
PATTERN_ID = '[?&]bookdetailid=([0-9]+)'


class Image:

    def __init__(self, url, page, p):
        self.url = LazyUrl(page.url, lambda _: url, self)
        ext = '.jpg'
        name = '{:04}{}'.format(p, ext)
        self.filename = '{}/{}'.format(page.title, name)


class Page:

    def __init__(self, title, url):
        self.title = clean_title(title)
        self.url = url
        self.id = int(re.find(PATTERN_ID, url))



class Downloader_jmana(Downloader):
    type = 'jmana'
    URLS = ['regex:'+PATTERN_ALL]
    MAX_CORE = 8
    _soup = None

    def init(self):
        self.url = clean_url(self.url)
        self.session = Session()
        if re.search(PATTERN_ID, self.url): #1799
            select = self.soup.find('select', class_='bookselect')
            for i, op in enumerate(select.findAll('option')[::-1]):
                if 'selected' in op.attrs:
                    break
            else:
                raise Exception('no selected option')
            for a in self.soup.findAll('a'):
                url = urljoin(self.url, a.get('href') or '')
                if re.search(PATTERN, url):
                    break
            else:
                raise Exception('list not found')
            self.url = self.fix_url(url)
            self._soup = None

            for i, page in enumerate(get_pages(self.url, self.soup, self.session)):
                if page.id == int(op['value']):
                    break
            else:
                raise Exception('can not find page')
            self.cw.range_p = [i]

    @classmethod
    def fix_url(cls, url):
        return url

    @property
    def soup(self):
        if self._soup is None:
            res = clf2.solve(self.url, session=self.session) #4070
            html = res['html']
            soup = Soup(html)
            self._soup = soup
        return self._soup

    @property
    def name(self):
        title = get_title(self.soup)
        artist = get_artist(self.soup)
        title = fix_title(self, title, artist)
        return title

    def read(self):
        title = self.name
        artist = get_artist(self.soup)
        self.artist = artist
        for img in get_imgs(self.url, title, self.session, soup=self.soup, cw=self.cw):
            if isinstance(img, Image):
                self.urls.append(img.url)
            else:
                self.urls.append(img)

        self.title = self.name



def get_title(soup):
    a = soup.find('a', class_='tit')
    if a:
        return a.text.strip()
    return re.find(r'제목\s*:\s*(.+)', soup.find('a', class_='tit').text, err='no title')


def get_artist(soup):
    return re.find(r'작가\s*:\s*(.+)', soup.text, default='').strip() or 'N/A'


@try_n(4, sleep=60)
def get_imgs_page(page, referer, session, cw=None):
    print_ = get_print(cw)
    sleep(5, cw) #2017
    html = downloader.read_html(page.url, referer, session=session)

    inserted = re.find(r'''var\s*inserted\s*=\s*['"](.*?)['"]''', html)
    print_('inserted: {}'.format(inserted))

    inserted = set(int(i) for i in inserted.split(',')) if inserted else set()

    soup = Soup(html)

    view = soup.find(class_='pdf-wrap')

    imgs = []
    for i, img in enumerate(child for child in view.children if isinstance(child, bs4.element.Tag)):
        src = img.get('data-src') or img.get('src') or ''

        if i in inserted:
            print_('remove: {}'.format(src))
            continue

        if not src:
            continue
        src = urljoin(page.url, src.strip())
        if '/adimg/' in src:
            print('adimg:', src)
            continue
        if '/notice' in src:
            print('notice:', src)
            continue

        img = Image(src, page, len(imgs))
        imgs.append(img)

    return imgs


def get_pages(url, soup, session):
    pages = []
    for inner in soup.findAll('div', class_='inner'):
        a = inner.find('a')
        if not a:
            continue
        href = a.attrs.get('href', '')
        if not re.search(PATTERN_ID, href):
            continue
        if a.find('img'):
            print('skip img', a.attrs.get('href'))
            continue
        href = urljoin(url, href)
        title_page = a.text
        page = Page(title_page, href)
        pages.append(page)

    pages = list(reversed(pages))
    return pages


@page_selector.register('jmana')
def f(url, win):
    if re.search(PATTERN_ID, url):
        raise Exception(tr_('목록 주소를 입력해주세요'))
    session = Session()
    res = clf2.solve(url, session=session, win=win) #4070
    soup = Soup(res['html'])
    pages = get_pages(url, soup, session)
    return pages


def get_imgs(url, title, session, soup=None, cw=None):
    print_ = get_print(cw)
    if soup is None:
        html = downloader.read_html(url, session=session)
        soup = Soup(html)
    pages = get_pages(url, soup, session)
    print_('pages: {}'.format(len(pages)))
    pages = page_selector.filter(pages, cw)
    imgs = []
    for i, page in enumerate(pages):
        check_alive(cw)
        imgs_already = get_imgs_already('jmana', title, page, cw)
        if imgs_already:
            imgs += imgs_already
            continue

        imgs += get_imgs_page(page, url, session, cw)
        if cw is not None:
            cw.setTitle('{} {} / {}  ({} / {})'.format(tr_('읽는 중...'), title, page.title, i + 1, len(pages)))

    if not imgs:
        raise Exception('no imgs')

    return imgs



File: src/extractor/navertoon_downloader.py

import downloader
from utils import Soup, urljoin, Downloader, get_imgs_already, clean_title, get_ext, get_print, errors, check_alive, File, try_n, json
import ree as re
import page_selector
from translator import tr_
import utils


class Page:

    def __init__(self, url, title, p):
        self.url = url
        self.title = title
        self.p = p


class File_navertoon(File):
    type = 'navertoon'
    format = 'title/page:04;'

    def __init__(self, info):
        ext = get_ext(info['url'])
        d = {
            'title': clean_title(info['title']),
            'page': info['page'],
            'chapterid': re.find(r'[?&]no=([0-9]+)', info['referer']), #6380
            }
        info['name'] = utils.format('navertoon', d, ext)

        super().__init__(info)


class Info:

    def __init__(self, id, title, artist):
        self.id = id
        self.title = title
        self.artist = artist



class Downloader_navertoon(Downloader):
    type = 'navertoon'
    URLS = ['comic.naver.com']
    MAX_CORE = 8
    MAX_SPEED = 4.0
    display_name = 'Naver Webtoon'
    ACCEPT_COOKIES = [r'(.*\.)?naver\.com']

    def init(self):
        self.__info, _ = get_pages(self.url, self.cw)

    @classmethod
    def fix_url(cls, url):
        url = re.sub(r'[?&]page=[0-9]+', '', re.sub(r'[?&]no=[0-9]+', '', url)).replace('m.comic.naver.', 'comic.naver.')
        url = url.replace('detail.nhn', 'list.nhn').replace('/detail?', '/list?')
        return url.rstrip('#')

    @property
    def name(self):
        id = self.__info.id
        title = self.__info.title
        artist = self.__info.artist
        title = self.format_title('N/A', id, title, artist, 'N/A', 'N/A', 'Korean', prefix='navertoon_')
        return clean_title(title)

    def read(self):
        self.title = tr_('읽는 중... {}').format(self.name)
        imgs = get_imgs_all(self.url, self.name, cw=self.cw)
        for img in imgs:
            self.urls.append(img)

        self.title = self.name


def set_no(url, p):
    if '&no=' not in url:
        url = url + f'&no={p}'
        return url
    url = re.sub('&no=[0-9]+', f'&no={p}', url)
    return url


def get_id(url):
    return int(url.lower().split('titleid=')[1].split('&')[0])


def set_page(url, p):
    if '&page=' in url:
        url = re.sub('&page=[0-9]+', f'&page={p}', url)
    else:
        url += f'&page={p}'
    return url


@try_n(4)
def get_pages(url, cw=None):
    print_ = get_print(cw)
    url = Downloader_navertoon.fix_url(url).replace('comic.naver.', 'm.comic.naver.')
    id = get_id(url)
    print('id:', id)
    print(url)
    html = downloader.read_html(url)
    soup = Soup(html)
    if soup.find('button', class_='btn_check'):
        raise errors.LoginRequired()
    try:
        info = soup.find('div', class_='area_info')
        artist = info.find('span', class_='author').text.strip()
    except Exception as e:
        print(e)
        try:
            title = ('\n').join(soup.find('div', class_='title').text.strip().split('\n')[:-1]).strip()
        except:
            title = 'artist not found'

        raise Exception(title)

    print_('artist: {}'.format(artist))
    title = soup.find('meta', {'property': 'og:title'}).attrs['content']
    pages = []
    nos = set()
    for p in range(1, 100):
        if p == 1:
            url_page = url
        else:
            url_page = set_page(url, p)
            html = downloader.read_html(url_page)
        print('read page:', url_page)
        soup = Soup(html)
        view = soup.findAll('ul', class_='section_episode_list')[(-1)]
        for lst in view.findAll('li'):
            url_page = urljoin(url, lst.find('a').attrs['href'])
            if 'detail.nhn' not in url_page.lower() and 'detail?' not in url_page.lower(): #3540
                continue
            print_('url_page: {}'.format(url_page))
            text = lst.find('strong', class_='title').find('span', class_='name').text.strip()
            no = int(re.findall('[?&]no=([0-9]+)', url_page)[0])
            if no in nos:
                print('duplicate no: {}'.format(no))
                continue
            nos.add(no)
            text = '{:04} - {}'.format(no, text)
            page = Page(url_page, text, p)
            pages.append(page)

        btn_next = soup.find('a', class_='btn_next')
        if btn_next is None or btn_next.attrs['href'] == '#':
            print('end of page')
            break

    info = Info(id, title, artist)
    return (
     info, pages)


@page_selector.register('navertoon')
@try_n(4)
def f(url):
    url = Downloader_navertoon.fix_url(url)
    info, pages = get_pages(url)
    return pages


@try_n(6)
def get_imgs(page, cw=None):
    print_ = get_print(cw)
    html = downloader.read_html(page.url)
    soup = Soup(html)

    type_ = re.find('''webtoonType *: *['"](.+?)['"]''', html)
    print_('type: {}'.format(type_))

    imgs = []
    if type_ == 'DEFAULT': # https://m.comic.naver.com/webtoon/detail.nhn?titleId=715772
        view = soup.find('div', class_='toon_view_lst')
        for img in view.findAll('img'):
            img = img.attrs.get('data-src')
            if not img:
                continue
            img = urljoin(page.url, img)
            img = File_navertoon({'referer': page.url, 'url':img, 'title': page.title, 'page': len(imgs)})
            imgs.append(img)
    elif type_ == 'CUTTOON': # https://m.comic.naver.com/webtoon/detail.nhn?titleId=752803
        view = soup.find('div', class_='swiper-wrapper')
        for div in view.findAll('div', class_='swiper-slide'):
            if div.parent != view:
                continue
            if div.find('div', class_='cut_viewer_last'):
                print('cut_viewer_last')
                continue
            if div.find('div', class_='cut_viewer_recomm'):
                print('cut_viewer_recomm')
                continue
            img = div.find('img')
            img = img.attrs['data-src']
            img = urljoin(page.url, img)
            img = File_navertoon({'referer': page.url, 'url':img, 'title': page.title, 'page': len(imgs)})
            imgs.append(img)
    elif type_ == 'EFFECTTOON': #2313; https://m.comic.naver.com/webtoon/detail.nhn?titleId=670144
        img_base = re.find('''imageUrl *: *['"](.+?)['"]''', html) + '/'
        print('img_base:', img_base)
        url_api = re.find('''documentUrl *: *['"](.+?)['"]''', html)
        data_raw = downloader.read_html(url_api, page.url)
        data = json.loads(data_raw)
        for img in data['assets']['stillcut'].values(): # ordered in python3.7+
            img = urljoin(img_base, img)
            img = File_navertoon({'referer': page.url, 'url':img, 'title': page.title, 'page': len(imgs)})
            imgs.append(img)
    else:
        _imgs = re.findall('sImageUrl *: *[\'"](.+?)[\'"]', html)
        if not _imgs:
            raise Exception('no imgs')
        for img in _imgs:
            img = urljoin(page.url, img)
            img = File_navertoon({'referer': page.url, 'url':img, 'title': page.title, 'page': len(imgs)})
            imgs.append(img)

    return imgs


def get_imgs_all(url, title, cw=None):
    print_ = get_print(cw)
    info, pages = get_pages(url, cw)
    pages = page_selector.filter(pages, cw)
    imgs = []
    for p, page in enumerate(pages):
        check_alive(cw)
        imgs_already = get_imgs_already('navertoon', title, page, cw)
        if imgs_already:
            imgs += imgs_already
            continue
        imgs_new = get_imgs(page, cw)
        print_('{}: {}'.format(page.title, len(imgs_new)))
        imgs += imgs_new
        if cw is not None:
            cw.setTitle(tr_('읽는 중... {} / {}  ({}/{})').format(title, page.title, p + 1, len(pages)))

    return imgs



File: src/extractor/yandere_downloader.py

from utils import Downloader, urljoin, clean_title, try_n, check_alive, LazyUrl, get_ext, get_max_range, limits
from translator import tr_
import ree as re
import downloader
from urllib.parse import unquote


@try_n(4)
@limits(.25)
def read_soup(url):
    return downloader.read_soup(url)



class Downloader_yandere(Downloader):
    type = 'yande.re'
    URLS = ['yande.re']
    MAX_CORE = 4
    ACCEPT_COOKIES = [r'(.*\.)?yande\.re']

    @classmethod
    def fix_url(cls, url):
        url = re.sub(r'([?&])page=[0-9]+&?', r'\1', url).rstrip('?&')
        pool = re.find('/pool/show/([0-9]+)', url)
        if pool is not None:
            url = urljoin(url, '/post?tags=pool%3A{}'.format(pool))
        return url

    def read(self):
        title = self.get_title(self.url)

        url = self.url
        n = get_max_range(self.cw)
        ids = set()
        while True:
            check_alive(self.cw)
            soup = read_soup(url)
            for a in soup.find_all('a', class_='thumb'):
                id_ = re.find(r'/show/([0-9]+)', a['href'], err='no id')
                if id_ in ids:
                    self.print_(f'dup: {id_}')
                    continue
                ids.add(id_)
                img = Image(urljoin(url, a['href']), id_)
                self.urls.append(img.url)
            if len(self.urls) >= n:
                del self.urls[n:]
                break

            self.cw.setTitle('{}  {} - {}'.format(tr_('읽는 중...'), title, len(self.urls)))

            next_page = soup.find('a', attrs={'rel':'next'}, href=True)
            if not next_page:
                break
            else:
                url = urljoin(self.url, next_page['href'])

        self.title = title

    def get_id(self, url:str) -> str:
        id_ = url.split('yande.re%20')[1].split('%20')[0]
        return int(id_)

    def get_title(self, url:str) -> str:
        if "tags=" not in url:
            raise NotImplementedError('no tags')

        url_tags = url.split("tags=")[-1].split('+')

        return clean_title(unquote(" ".join(url_tags)))


class Image:

    def __init__(self, url, id_):
        self._id = id_
        self.url = LazyUrl(url, self.get, self)

    def get(self, url):
        soup = read_soup(url)
        img = soup.find('a', class_='original-file-unchanged') or soup.find('a', class_='original-file-changed')
        img = urljoin(url, img['href'])
        ext = get_ext(img)
        self.filename = clean_title(self._id, n=-len(ext)) + ext
        return img



File: src/extractor/imgur_downloader.py

import downloader
from utils import Downloader, Soup, try_n, urljoin, get_max_range, clean_title, cut_pair, check_alive, json
import ree as re
import os
from translator import tr_



class Downloader_imgur(Downloader):
    type = 'imgur'
    URLS = ['imgur.com']
    MAX_CORE = 16
    ACCEPT_COOKIES = [r'(.*\.)?imgur\.com']

    def init(self):
        self.info = get_info(self.url)

    @property
    def id_(self):
        return re.find('imgur.com/.+?/([0-9a-zA-Z]+)', self.url)

    @property
    def name(self):
        title = self.info['title'] or 'N/A'
        return clean_title(title, n=100)

    def read(self):
        imgs = get_imgs(self.url, self.info, self.cw)
        for img in imgs:
            ext = os.path.splitext(img.split('?')[0])[1]
            if len(imgs) > 1:
                self.filenames[img] = '{:04}{}'.format(len(self.urls), ext)
            else:
                self.filenames[img] = clean_title(self.name, n=-len(ext)) + ext
            self.urls.append(img)

        self.single = len(imgs) == 1
        self.referer = self.url
        self.title = '{} (imgur_{})'.format(self.name, self.id_)


@try_n(4)
def get_info(url):
    url = url.replace('/gallery/', '/a/')
    if '/r/' in url and url.split('/r/')[1].strip('/').count('/') == 0:
        title = re.find(r'/r/([^/]+)', url)
        info = {}
        info['title'] = title
        info['type'] = 'r'
    else:
        try: # legacy
            html = downloader.read_html(url, cookies={'over18':'1'})
            s = re.find('image *: *({.+)', html)
            info_raw = cut_pair(s)
        except Exception as e: # new
            print(e)
            id_ = re.find(r'/a/([0-9a-zA-Z_]+)', url) or re.find(r'/r/[0-9a-zA-Z_]+/([0-9a-zA-Z_]+)', url, err='no id')
            url_api = 'https://api.imgur.com/post/v1/albums/{}?client_id=546c25a59c58ad7&include=media%2Cadconfig%2Caccount'.format(id_)
            info_raw = downloader.read_html(url_api, cookies={'over18':'1'})
        info = json.loads(info_raw)
        info['type'] = 'a'
    return info


def get_imgs(url, info=None, cw=None):
    print('get_imgs', url)
    if info is None:
        info = get_info(url)
    imgs = []

    # Range
    max_pid = get_max_range(cw)

    if info['type'] == 'a':
        if 'album_images' in info: # legacy
            imgs_ = info['album_images']['images']
        elif 'media' in info: # new
            imgs_ = info['media']
        else: # legacy
            imgs_ = [info]

        for img in imgs_:
            img_url = img.get('url') # new
            if not img_url: # legacy
                hash = img['hash']
                ext = img['ext']
                img_url = 'https://i.imgur.com/{}{}'.format(hash, ext)
            if img_url in imgs:
                continue
            imgs.append(img_url)

    elif info['type'] == 'r':
        urls = set()
        for p in range(100):
            url_api = 'https://imgur.com/r/{}/new/page/{}/hit?scrolled'.format(info['title'], p)
            print(url_api)
            html = downloader.read_html(url_api, referer=url)
            soup = Soup(html)

            c = 0
            for post in soup.findAll('div', class_='post'):
                check_alive(cw)
                a = post.find('a', class_='image-list-link')
                url_post = urljoin(url, a.attrs['href'])
                if url_post in urls:
                    continue
                urls.add(url_post)
                c += 1

                try: # for r18 images
                    imgs += get_imgs(url_post)
                except Exception as e:
                    print(e)

                s = '{} {}  ({})'.format(tr_('읽는 중...'), info['title'], len(imgs))
                if cw is not None:
                    cw.setTitle(s)
                else:
                    print(s)

            if c == 0:
                print('same; break')
                break

            if len(imgs) >= max_pid:
                break

    return imgs



File: src/extractor/navercafe_downloader.py

#coding:utf8
from utils import Downloader, get_print, urljoin, Soup, get_ext, File, clean_title, downloader, re, try_n, errors, json, Session
import utils


class LoginRequired(errors.LoginRequired):
    def __init__(self, *args):
        super().__init__(*args, method='browser', url='https://nid.naver.com/nidlogin.login')


class Downloader_navercafe(Downloader):
    type = 'navercafe'
    URLS = ['cafe.naver.com']
    display_name = 'Naver Cafes'
    ACCEPT_COOKIES = [r'(.*\.)?naver\.com']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        m = re.find(r'cafe\.naver\.com/([^/?#]+).+?articleid%3D([0-9]+)', url)
        if m:
            url = 'https://cafe.naver.com/{}/{}'.format(*m)
        return url

    def read(self):
        info = get_info(self.url, self.session, self.cw)
        for img in info['imgs']:
            self.urls.append(img)
        tail = f' ({info["cafename"]}_{info["id"]})'
        self.title = clean_title(info['title'], n=-len(tail)) + tail


@try_n(4)
def get_info(url, session, cw=None):
    print_ = get_print(cw)
    info = {}

    html = downloader.read_html(url, 'http://search.naver.com', session=session)
    soup = Soup(html)
    if '"cafe_cautionpage"' in html:
        raise LoginRequired()
    url_article = re.find(r'''//cafe\.naver\.com/ArticleRead\.nhn\?articleid=[0-9]+[^'"]*''', html, err='no articleid')
    url_article = urljoin(url, url_article)

    print_(url_article)

    articleid = re.find(r'articleid=([0-9]+)', url_article)
    clubid = re.find(r'clubid(=|%3D)([0-9]+)', url_article)[1]
    art = re.find(r'art=(.+?)&', url_article)
    if art:
        url_api = f'https://apis.naver.com/cafe-web/cafe-articleapi/v2.1/cafes/{clubid}/articles/{articleid}?art={art}&useCafeId=true&requestFrom=A'
    else:
        url_api = f'https://apis.naver.com/cafe-web/cafe-articleapi/v2.1/cafes/{clubid}/articles/{articleid}?query=&useCafeId=true&requestFrom=A'

    j = downloader.read_json(url_api, url_article, session=session)

    if j['result'].get('errorCode'): #6358
        raise LoginRequired(j['result'].get('reason'))

    info['title'] = j['result']['article']['subject']
    info['cafename'] = j['result']['cafe']['url']
    info['cafeid'] = clubid
    info['id'] = articleid

    html_content = j['result']['article']['contentHtml']
    soup = Soup(html_content)

    imgs = []

    pairs = []

    for video in soup.findAll('span', class_='_naverVideo'):
        vid = video.attrs['vid']
        key = video.attrs['key']
        pairs.append((vid, key))

    for script in soup.findAll('script', class_='__se_module_data'):
        data_raw = script['data-module']
        data = json.loads(data_raw)['data']
        vid = data.get('vid')
        if not vid:
            continue
        key = data['inkey']
        pairs.append((vid, key))

    for vid, key in pairs:
        url_api = f'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{vid}?key={key}'
        data_raw = downloader.read_html(url_api)
        data = json.loads(data_raw)
        fs = data['videos']['list']
        fs = sorted(fs, key=lambda f: f['size'], reverse=True)
        video = Image({'url': fs[0]['source'], 'referer': url_article, 'p': len(imgs)})
        imgs.append(video)

    for img in soup.findAll('img'):
        img = Image({'url': urljoin(url_article, img['src']), 'referer': url, 'p': len(imgs)})
        imgs.append(img)

    info['imgs'] = imgs

    return info


class Image(File):
    type = 'navercafe'
    format = 'page:04;'

    def __init__(self, info):
        self._url = info['url']
        info['url'] = re.sub(r'[?&]type=[wh0-9]+', '', self._url) #6460
        ext = get_ext(info['url'])
        d = {
            'page': info['p'],
            }
        info['name'] = utils.format('navercafe', d, ext)
        super().__init__(info)

    def alter(self):
        return self._url



File: src/extractor/hf_downloader.py

#coding:utf8
import downloader
from utils import Soup, urljoin, Session, LazyUrl, Downloader, try_n, clean_title, check_alive
import ree as re
import os
from translator import tr_
URL_ENTER = 'https://www.hentai-foundry.com/site/index?enterAgree=1&size=1550'
URL_FILTER = 'https://www.hentai-foundry.com/site/filters'


class Image:
    def __init__(self, url, session):
        @try_n(4)
        def f(_):
            html = downloader.read_html(url, session=session)
            soup = Soup(html)

            box = soup.find('section', id='picBox')
            img = box.find('img')
            if img is None:
                raise Exception('No img')

            onclick = img.attrs.get('onclick', '')
            if onclick and '.src' in onclick:
                print('onclick', onclick)
                img = re.find('''.src *= *['"](.+?)['"]''', onclick)
            else:
                img = img.attrs['src']
            img = urljoin(url, img)

            filename = clean_title(os.path.basename(img.split('?')[0]))
            name, ext = os.path.splitext(filename)

            # https://www.hentai-foundry.com/pictures/user/DrGraevling/74069/Eversong-Interrogation-pg.-13
            if ext.lower() not in ['.bmp', '.png', '.gif', '.jpg', '.jpeg', '.webp', '.webm', '.avi', '.mp4', '.mkv', '.wmv']:
                filename = '{}.jpg'.format(name)

            self.filename = filename
            return img
        self.url = LazyUrl(url, f, self)


def get_username(url):
    if 'user/' in url:
        username = url.split('user/')[1].split('?')[0].split('/')[0]
    return username



class Downloader_hf(Downloader):
    type = 'hf'
    URLS = ['hentai-foundry.com']
    MAX_CORE = 16
    display_name = 'Hentai Foundry'
    ACCEPT_COOKIES = [r'(.*\.)?hentai-foundry\.com']

    def init(self):
        self.session = enter()

    @classmethod
    def fix_url(cls, url):
        username = get_username(url)
        return 'https://www.hentai-foundry.com/user/{}'.format(username)

    def read(self):
        username = get_username(self.url)
        self.title = username

        imgs = get_imgs(username, self.title, self.session, cw=self.cw)

        for img in imgs:
            self.urls.append(img.url)

        self.title = username


@try_n(2)
def enter():
    print('enter')
    session = Session()

    r = session.get(URL_ENTER)

    # 862
    html = r.text
    soup = Soup(html)
    box = soup.find('aside', id='FilterBox')
    data = {}
    for select in box.findAll('select'):
        name = select.attrs['name']
        value = select.findAll('option')[-1].attrs['value']
        print(name, value)
        data[name] = value
    for input in box.findAll('input'):
        name = input.attrs['name']
        value = input.attrs['value']
        if name.startswith('rating_') or 'CSRF_TOKEN' in name:
            print(name, value)
            data[name] = value
    data.update({
        'filter_media': 'A',
        'filter_order': 'date_new',
        'filter_type': '0',
        })
    r = session.post(URL_FILTER, data=data, headers={'Referer': r.url})
    print(r)

    return session


def get_imgs(username, title, session, cw=None):
    url = 'https://www.hentai-foundry.com/pictures/user/{}'.format(username)

    #downloader.read_html(url_enter, session=session)

    hrefs = []
    for p in range(100):
        check_alive(cw)
        print(url)
        html = downloader.read_html(url, session=session)
        soup = Soup(html)

        if soup.find('div', id='entryButtonContainer'):
            session = enter()
            continue

        tab = soup.find('a', class_='active')
        n = re.find(r'\(([0-9]+)', tab.text)

        view = soup.find('div', class_='galleryViewTable')
        for a in view.findAll('a', class_='thumbLink'):
            href = urljoin(url, a.attrs['href'])
            if href in hrefs:
                print('dup')
                continue
            hrefs.append(href)

        next = soup.find(lambda tag: tag.name == 'li' and tag.get('class') == ['next'])
        if next is None:
            break
        url = urljoin(url, next.a.attrs['href'])

        s = '{}  {}  ({} / {})'.format(tr_('읽는 중...'), title, len(hrefs), n)
        if cw:
            cw.setTitle(s)
        else:
            print(s)

    imgs = []
    for href in hrefs:
        img = Image(href, session)
        imgs.append(img)

    return imgs



File: src/extractor/file_downloader.py

import downloader, os
from utils import Downloader, query_url, clean_title, get_ext, Session
from hashlib import md5



class Downloader_file(Downloader):
    type = 'file'
    single = True
    URLS = []

    def init(self):
        self.session = Session() #6525

    @classmethod
    def fix_url(cls, url):
        if url and '://' not in url:
            url = 'https://' + url.lstrip('/')
        return url

    def read(self):
        if not self.url.strip():
            raise Exception('empty url')
        qs = query_url(self.url)
        for key in qs:
            if key.lower() in ('file', 'filename'):
                name = qs[key][(-1)]
                break
        else:
            name = self.url
            for esc in ['?', '#']:
                name = name.split(esc)[0]
            name = os.path.basename(name.strip('/'))

        try:
            ext = downloader.get_ext(self.url)
        except:
            ext = ''
        if not ext:
            ext = get_ext(name)

        name = os.path.splitext(name)[0]

        self.urls.append(self.url)

        id_ = md5(self.url.encode('utf8')).hexdigest()[:8]
        tail = ' ({}){}'.format(id_, ext)
        filename = clean_title(name, n=-len(tail)) + tail

        self.filenames[self.url] = filename

        self.title = filename



File: src/extractor/wikiart_downloader.py

#coding:utf8
import downloader
from utils import LazyUrl, Downloader, Session, get_print, clean_title, check_alive, json
import os
from translator import tr_



class Image:
    def __init__(self, url, referer, title, id):
        self.url = LazyUrl(referer, lambda _: url, self)
        ext = os.path.splitext(url.split('?')[0])[1]
        n = len(id) + len(ext) + 3
        title = clean_title(title, n=-n)
        self.filename = '{} - {}{}'.format(id, title, ext)



class Downloader_wikiart(Downloader):
    type = 'wikiart'
    URLS = ['wikiart.org']
    display_name = 'WikiArt'
    ACCEPT_COOKIES = [r'(.*\.)?wikiart\.org']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        url = 'https://www.wikiart.org/en/{}'.format(get_id(url))
        return url

    def read(self):
        artist = get_artist(get_id(self.url), self.session)
        self.artist = artist

        for img in get_imgs(self.url, artist, self.session, cw=self.cw):
            self.urls.append(img.url)

        self.title = clean_title(artist)



def get_id(url):
    userid = url.split('?')[0].split('#')[0].split('wikiart.org/')[1].split('/')[1]
    return userid


def get_imgs(url, artist, session, cw=None):
    print_ = get_print(cw)
    userid = get_id(url)
    print(userid)

    imgs = []
    ids = set()
    for p in range(1, 100):
        check_alive(cw)
        url_api = 'https://www.wikiart.org/en/{}/mode/all-paintings?json=2&layout=new&page={}&resultType=masonry'.format(userid, p)
        print(url_api)
        data_raw = downloader.read_html(url_api, url, session=session)
        data = json.loads(data_raw)

        _imgs = data['Paintings']
        n = data['AllPaintingsCount']

        if not _imgs:
            print_('???')
            break

        for p in _imgs:
            img = p['image']
            id = p['id']
            referer = p['paintingUrl']
            title = p['title']
            if id in ids:
                print('duplicate: {}'.format(id))
                continue
            ids.add(id)
            img = Image(img, referer, title, id)
            imgs.append(img)

        s = '{}  {} - {} / {}'.format(tr_('읽는 중...'), artist, len(imgs), n)
        if cw:
            cw.setTitle(s)
        else:
            print(s)

        if len(imgs) == n:
            print_('full')
            break

    return imgs


def get_artist(userid, session):
    url = 'https://www.wikiart.org/en/{}'.format(userid)
    soup = downloader.read_soup(url, session=session)

    return soup.find('h3').text.strip()



File: src/extractor/pinter_downloader.py

from utils import Session, Downloader, LazyUrl, clean_url, try_n, clean_title, get_ext, get_max_range, check_alive, limits, json
import ree as re
from translator import tr_
import urllib
from m3u8_tools import playlist2stream, M3u8_stream
BASE_URL = 'https://www.pinterest.com'



class Downloader_pinter(Downloader):
    type = 'pinter'
    URLS = ['pinterest.']
    type_pinter = 'board'
    display_name = 'Pinterest'
    ACCEPT_COOKIES = [r'(.*\.)?(pinterest)\.']

    @try_n(4)
    def init(self):
        self.session = Session('chrome')
        self.api = PinterestAPI(self.session)
        self._pin_id = re.find(r'https?://.*pinterest\.[^/]+/pin/([0-9]+)', self.url)
        if self._pin_id is not None:
            self.type_pinter = 'pin'
        else:
            username, board = get_username_board(self.url)
            if '/' in board:
                self.type_pinter = 'section'
            if board == '_created':
                self.type_pinter = 'created'
        self.print_(f'type: {self.type_pinter}')
        if self.type_pinter in ['board', 'section', 'created']:
            self.info = get_info(username, board, self.api)
        elif self.type_pinter == 'pin':
            pass #5132
        else:
            raise NotImplementedError(self.type_pinter)

    @classmethod
    def fix_url(cls, url):
        if 'pinterest.' not in url:
            url = f'https://www.pinterest.com/{url}'
        return url

    @property
    def name(self):
        if self.type_pinter == 'pin':
            return self._pin_id
        username = ''
        name = ''
        if self.type_pinter == 'created':
            username = self.info['native_creator']['username']
            name = '_created'
        else:
            username = self.info['owner']['username']
            name = self.info['name']
        return clean_title(f'{username}/{name}')

    def read(self):
        if self.type_pinter == 'pin':
            self.single = True
            id = self._pin_id
        else:
            id = self.info['id']
        self.title = self.name
        imgs = get_imgs(id, self.api, cw=self.cw, title=self.name, type=self.type_pinter)
        for img in imgs:
            self.urls.append(img.url)
        self.title = self.name


def get_info(username, board, api):
    if '/' in board:
        section = '/'.join(board.split('/')[1:])
        board = board.split('/')[0]
        info = api.board(username, board)
        for s in api.board_sections(info['id']):
            print(s['slug'].lower(), section)
            if s['slug'].lower() == section.lower():
                break
        else:
            raise Exception('Invalid section')

        title = s['title']
        info.update(s)
        info['name'] = f'{info["name"]}/{title}'
        print('section_id:', info['id'])
    elif board == '_created':
        info = api.board_created(username)[0]
    else:
        info = api.board(username, board)
    return info


class PinterestAPI:
    HEADERS = {
        'Accept': 'application/json, text/javascript, */*, q=0.01',
        'Accept-Language': 'en-US,en;q=0.5',
        'Referer': BASE_URL + '/',
        'X-Requested-With': 'XMLHttpRequest',
        'X-APP-VERSION' : '31461e0',
        'X-Pinterest-AppState': 'active',
        'Origin': BASE_URL,
        }

    def __init__(self, session):
        self.session = session
        self.session.headers.update(self.HEADERS)

    def pin(self, pin_id):
        options = {'id': pin_id, 'field_set_key': 'detailed'}
        return self._call('Pin', options)['resource_response']['data']

    def pin_related(self, pin_id):
        options = {'pin': pin_id, 'add_vase': True, 'pins_only': True}
        return self._pagination('RelatedPinFeed', options)

    def board(self, user, board):
        options = {'slug': board, 'username': user, 'field_set_key': 'detailed'}
        return self._call('Board', options)['resource_response']['data']

    def board_pins(self, board_id):
        options = {'board_id': board_id}
        return self._pagination('BoardFeed', options)

    def board_related(self, board_id):
        options = {'board_id': board_id, 'add_vase': True}
        return self._pagination('BoardRelatedPixieFeed', options)

    def board_sections(self, board_id):
        options = {'board_id': board_id}
        return self._pagination('BoardSections', options)

    def board_section_pins(self, section_id):
        options = {'section_id': section_id}
        return self._pagination('BoardSectionPins', options)

    def board_created(self, user):
        options = {'data': {}, 'username': user, 'field_set_key': 'grid_item'}
        return self._call('UserActivityPins', options)['resource_response']['data']

    def board_created_pins(self, user):
        options = {'data': {}, 'username': user, 'field_set_key': 'grid_item'}
        return self._pagination('UserActivityPins', options)

    @try_n(4)
    @limits(4) # 1000 calls per hour
    def _call(self, resource, options):
        url = f'{BASE_URL}/resource/{resource}Resource/get/'
        params = {'data': json.dumps({'options': options}), 'source_url': ''}
        #print(f'_call: {url}, {params}')
        r = self.session.get(url, params=params)
        s = r.text
        status_code = r.status_code
        try:
            data = json.loads(s)
        except ValueError:
            data = {}

        if status_code < 400 and not r.history:
            return data

        if status_code == 404 or r.history:
            raise Exception('Not Found')
        raise Exception(f'API request failed: {status_code}')

    def _pagination(self, resource, options):
        while True:
            data = self._call(resource, options)
            if resource == 'UserActivityPins' and len(data['resource_response']['data']) == 0:
                return
            for x in data['resource_response']['data']:
                yield x

            try:
                bookmarks = data['resource']['options']['bookmarks']
                if not bookmarks or bookmarks[0] == '-end-' or bookmarks[0].startswith('Y2JOb25lO'):
                    return
                options['bookmarks'] = bookmarks
            except KeyError:
                return


class Image:

    def __init__(self, img):
        self.id = img['id']
        print(self.id)
        videos = img.get('videos')
        if videos and 'video_list' in videos:
            src = list(videos['video_list'].values())[0]['url']
        else:
            src = img['images']['orig']['url']

        ext = get_ext(src)
        if ext.lower() == '.m3u8':
            try:
                src = playlist2stream(src)
            except:
                src = M3u8_stream(src)
            ext = '.mp4'

        self.url = LazyUrl(f'{BASE_URL}/pin/{self.id}/', lambda _: src, self)
        self.filename = f'{self.id}{ext}'



def get_imgs(id, api, cw=None, title=None, type='board'):
    n = get_max_range(cw)
    imgs = []
    ids = set()
    print(f'get_imgs: type={type}')
    if type == 'board':
        gen = api.board_pins(id)
    elif type == 'section':
        gen = api.board_section_pins(id)
    elif type == 'pin':
        gen = [api.pin(id)]
    elif type == 'created':
        gen = api.board_created_pins(title.split('／')[0])
    else:
        raise Exception(f'Type "{type}" is not supported')
    for img in gen:
        check_alive(cw)
        if 'images' not in img:
            print('skip img:', img['id'])
            continue
        img = Image(img)
        if type == 'pin' and img.id != id:
            raise AssertionError('id mismatch')
        if img.id in ids:
            print('duplicate:', img.id)
            continue
        ids.add(img.id)
        print(img.url)
        print(img.filename)
        print()
        imgs.append(img)
        if len(imgs) >= n:
            break
        if cw is not None:
            cw.setTitle('{} {}  ({})'.format(tr_('읽는 중...'), title, len(imgs)))

    return imgs


def get_username_board(url):
    url = clean_url(url)
    m = re.search('pinterest.[a-zA-Z.]+?/([^/]+)/([^#\\?]+)', url)
    username, board = m.groups()
    board = urllib.parse.unquote(board).strip()
    while board.endswith('/'):
        board = board[:-1].strip()

    return (username, board)



File: src/extractor/pawoo_downloader.py

#coding:utf8
from utils import Downloader, clean_title, Session, Soup, urljoin
import clf2
from mastodon import get_info
import ree as re



def get_id(url):
    return re.find('pawoo.net/([^/]+)', url.lower())



class Downloader_pawoo(Downloader):
    type = 'pawoo'
    URLS = ['pawoo.net']
    ACCEPT_COOKIES = [r'(.*\.)?pawoo\.net']

    def init(self):
        self.session = Session()
        if get_id(self.url) == 'web': #6123
            soup = Soup(clf2.solve(self.url)['html'])
            name = soup.find('div', class_='account__header__tabs__name')
            id_ = name.find('small').text.strip()
            self.url = urljoin(self.url, f'/{id_}')

    @classmethod
    def fix_url(cls, url):
        if url.endswith('/media'):
            url = url[:-len('/media')]
        id_ = get_id(url) or url
        if id_ == 'web':
            return url
        return f'https://pawoo.net/{id_}'

    def read(self):
        id_ = get_id(self.url)
        info = get_info('pawoo.net', id_, f'pawoo_{id_}', self.session, self.cw)

        self.urls += info['files']

        self.title = clean_title('{} (pawoo_{})'.format(info['title'], id_))



File: src/extractor/manatoki_downloader.py

from utils import Soup, try_n, Downloader, urljoin, get_print, Session, clean_title, get_ext, fix_title, lazy, get_imgs_already, check_alive, File, limits
from translator import tr_
import page_selector
import utils
import clf2
import ree as re
from PIL import Image


class File_manatoki(File):
    type = 'manatoki'
    format = 'title/page:04;'
    show_pp = False

    def __init__(self, info):
        ext = get_ext(info['url'])
        if ext.lower()[1:] not in ['jpg', 'jpeg', 'bmp', 'png', 'gif', 'webm', 'webp']:
            ext = '.jpg'
        d = {
            'title': info['title'],
            'page': info['page'],
            'chapterid': re.find(r'/comic/([0-9]+)', info['referer']), #6380
            }
        info['name'] = utils.format('manatoki', d, ext)

        super().__init__(info)

    @limits(.5)
    def get(self):
        return {}

    def pp(self, filename): #5233
        img = Image.open(filename)
        nf = getattr(img, 'n_frames', 1)
        loop = img.info.get('loop')
        if nf > 1 and loop:
            img.seek(nf-1)
            img.save(filename)
        img.close()
        return filename



class Page:
    def __init__(self, title, url):
        self.title = clean_title(title)
        self.url = url
        self.id = int(re.find(r'/(comic|webtoon)/([0-9]+)', url, err='no id')[1])



class Downloader_manatoki(Downloader):
    type = 'manatoki'
    URLS = [r'regex:(mana|new)toki[0-9]*\.(com|net)']
    MAX_CORE = 4
    ACCEPT_COOKIES = [r'(.*\.)?(mana|new)toki[0-9]*\.(com|net)']

    @try_n(2)
    def init(self):
        self.session, self.soup, url = get_soup(self.url, cw=self.cw)
        self.url = self.fix_url(url)

        # 2377
        list = self.soup.find(attrs={'data-original-title': '목록'})
        if list:
            url = urljoin(self.url, list.parent['href'])
            nav = self.soup.find('div', class_='toon-nav')
            select = nav.find('select', {'name': 'wr_id'})
            for i, op in enumerate(select.findAll('option')[::-1]):
                if 'selected' in op.attrs:
                    break
            else:
                raise Exception('no selected option')
            self.session, self.soup, url = get_soup(url, cw=self.cw)
            url_page = self.fix_url(url)

            for i, page in enumerate(get_pages(url_page, self.soup)):
                if page.id == int(op['value']):
                    break
            else:
                raise Exception('can not find page')
            self.cw.range_p = [i]
            self.url = url_page

        self.name

    @classmethod
    def fix_url(cls, url):
        # 2377
        m = re.find(r'/board.php\?bo_table=([0-9a-zA-Z_]+)&wr_id=([0-9]+)', url)
        if m:
            return urljoin(url, '/{}/{}'.format(*m))
        return url.split('?')[0]

    @classmethod
    def key_id(cls, url):
        return '/'.join(url.split('/')[3:5])

    @lazy
    def name(self):
        artist = get_artist(self.soup)
        title = self.soup.find('meta', {'name':'subject'})['content'].strip()
        return fix_title(self, title, artist)

    def read(self):
        self.title = tr_('읽는 중... {}').format(self.name)
        self.artist = get_artist(self.soup)

        imgs = get_imgs(self.url, self.name, self.soup, self.session, self.cw)

        for img in imgs:
            self.urls.append(img)

        self.title = self.name



def get_artist(soup):
    view = soup.find('div', class_='view-title', err='no title')
    text = view.text.replace('\n', '#')
    artist = re.find(r'작가[ #]*:[ #]*(.+?)#', text, default='N/A').strip()
    return artist


@limits(10)
def get_soup(url, session=None, cw=None, win=None):
    if session is None:
        session = Session()
    virgin = True
    def f(html, browser=None):
        nonlocal virgin
        soup = Soup(html)
        if soup.find('form', {'name':'fcaptcha'}): #4660
            browser.show()
            if virgin:
                virgin = False
                browser.runJavaScript('window.scrollTo({top: document.getElementsByClassName("form-box")[0].getBoundingClientRect().top-150})') #5504
            return False
        browser.hide()
        return True
    res = clf2.solve(url, session=session, f=f, cw=cw, win=win)
    soup = Soup(res['html'], apply_css=True)

    return session, soup, res['url']


def get_pages(url, soup, sub=False):
    list = soup.find('ul', class_='list-body')
    pages = []
    for item in list.findAll('div', 'wr-subject'):
        for span in item.a.findAll('span'):
            span.decompose()
        title = item.a.text.strip()
        href = item.a['href']
        href = urljoin(url, href)
        pages.append((title, href))

    if not pages:
        raise Exception('no pages')

##    if sub: #4909
##        return pages
##    else:
##        pg = soup.find('ul', class_='pagination')
##        as_ = pg.findAll('a')
##        for a in as_:
##            href = a.get('href')
##            if not href:
##                continue
##            href = urljoin(url, href)
##            for try_ in range(2):
##                try:
##                    session, soup2, href = get_soup(href)
##                    pages += get_pages(href, soup2, sub=True)
##                    break
##                except Exception as e:
##                    e_ = e
##                    print(e)
##            else:
##                raise e_

    titles = {}
    pages_ = []
    for title, href in pages[::-1]:
        title = utils.fix_dup(title, titles) #4161
        page = Page(title, href)
        pages_.append(page)

    return pages_


@page_selector.register('manatoki')
def f(url, win):
    session, soup, url = get_soup(url, win=win)
    list = soup.find('ul', class_='list-body')
    if list is None:
        raise Exception(tr_('목록 주소를 입력해주세요'))
    pages = get_pages(url, soup)
    return pages


def get_imgs(url, title, soup=None, session=None, cw=None):
    print_ = get_print(cw)

    if soup is None or session is None:
        session, soup, url = get_soup(url, session, cw)

    pages = get_pages(url, soup)
    pages = page_selector.filter(pages, cw)

    imgs = []
    for i, page in enumerate(pages):
        check_alive(cw)
        imgs_already = get_imgs_already('manatoki', title, page, cw)
        if imgs_already:
            imgs += imgs_already
            continue

        imgs_ = get_imgs_page(page, title, url, session, cw)
        imgs += imgs_

        s = '{} {} / {}  ({} / {})'.format(tr_('읽는 중...'), title, page.title, i+1, len(pages))
        print_('{} {}'.format(page.title, len(imgs_)))
        if cw is not None:
            cw.setTitle(s)
        else:
            print('read page... {}    ({})'.format(page.url, len(imgs)))

    return imgs


@try_n(4)
def get_imgs_page(page, title, referer, session, cw):
    print_ = get_print(cw)

    # 2183
    session, soup, page.url = get_soup(page.url, session, cw)

    title_page = page.title#clean_title(soup.find('span', class_='page-desc').text.strip())
    if page.title != title_page:
        print_('{} -> {}'.format(page.title, title_page))
        page.title = title_page

    views = soup.findAll('div', class_='view-content')\
            + soup.findAll('div', class_='view-padding')
    if not views:
        raise Exception('no views')

    hash = re.find(r'''data_attribute\s*:\s*['"](.+?)['"]''', soup.html)
    print_('hash: {}'.format(hash))
    if hash is None:
        raise Exception('no hash')

    imgs = []
    for view in views:
        if view is None:
            continue
        for img in view.findAll('img'):
            if not isVisible(img):
                continue
            src = img.get('data-{}'.format(hash))
            src = src or img.get('content') # https://manatoki77.net/comic/5266935
            if not src:
                continue
            img = urljoin(page.url, src)
            if '/img/cang' in img:
                continue
            if '/img/blank.gif' in img:
                continue
            img = File_manatoki({'referer': page.url, 'url': img, 'title': page.title, 'page': len(imgs)})
            imgs.append(img)

##    if not imgs:
##        raise Exception('no imgs')

    return imgs


def isVisible(tag):
    while tag:
        if re.search(r'display:\s*none', tag.get('style', ''), re.I):
            return False
        tag = tag.parent
    return True



File: src/extractor/talk_op_gg_downloader.py

# coding: UTF-8
# title: Download talk op.gg image
# author: SaidBySolo
# comment: op.gg 커뮤니티의 이미지를 다운로드합니다

"""
MIT License

Copyright (c) 2020 SaidBySolo

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import requests
from utils import Downloader, Soup



class DownloaderTalkOPGG(Downloader):
    type = "talkopgg"
    URLS = ["talk.op.gg"]

    def init(self) -> None:
        pass

    def read(self) -> None:
        response = requests.get(self.url)
        soup = Soup(response.text)

        self.title = soup.find("title").text

        image_element_list = soup.find("div", class_="article-content").findAll("img")

        for image_element in image_element_list:
            self.urls.append(image_element["src"])



File: src/extractor/newgrounds_downloader.py

# coding:utf8
from datetime import datetime
import downloader
import errors
from functools import reduce
import os
import ree as re
from timee import sleep
from translator import tr_
from utils import Downloader, clean_title, Session, get_print, try_n, check_alive


class Downloader_newgrounds(Downloader):
    type = 'newgrounds'
    URLS = ['newgrounds.com']
    ACCEPT_COOKIES = [r'(.*\.)?newgrounds\.com']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        user = re.find(r'(?:http(?:s)?://)?([^\.]+).newgrounds.com', url.lower())
        if not user or user == 'www':
            user = re.find(r'newgrounds.com/art/view/([^/?#]+)', url, err='no user id')
        return 'https://{}.newgrounds.com/art'.format(user)

    def read(self):
        user = re.find('(?:http(?:s)?://)?([^\.]+).newgrounds.com', self.url.lower())
        title = clean_title(user)

        for img in get_imgs(user, title, self.session, self.cw):
            self.urls.append(img.url)
            self.filenames[img.url] = img.filename

        self.title = title


class Image:
    def __init__(self, url, filename=None):
        self.url = url
        if filename is None:
            filename = os.path.basename(url)
        self.filename = filename


@try_n(10, sleep=20)
def get_posts(url, params, session, print_):
    posts, data, items = [], None, None
    try:
        data = session.get(url, params=params).json()
        items = data.get('items')
        if items:
            for item in reduce(lambda x, y: x + y, items.values()):
                posts.append(re.find('(?<=href=")([^"]+)', item))
    except Exception as e:
        print_('failed to get posts')
        print_('no. posts: {}'.format(len(posts)))
        print_('data: {}'.format(data))
        print_('items: {}'.format(items))
        raise e

    return posts


@try_n(10, sleep=20)
def get_html(post, session):
    return downloader.read_html(post, session=session)


def get_img(post, session, print_):
    html, url, name, ext, _datetime = None, None, None, None, None
    try:
        html = get_html(post, session)
        if 'You must be logged in, and at least 18 years of age to view this content!' in html:
            raise errors.LoginRequired()
        url = re.find('(?<="full_image_text":"<img src=\\\\")([^"]+)', html).replace('\\', '')
        name = re.find('(?<=alt=\\\\")([^\\\\]+)', html)
        ext = os.path.splitext(url)[1].split('?')[0]
        _datetime = datetime.strptime(re.find('(?<="datePublished" content=")([^"]+)', html), '%Y-%m-%dT%H:%M:%S%z')
    except Exception as e:
        print_('failed to get images')
        print_('post: {}'.format(post))
        print_('url: {}'.format(url))
        print_('name: {}'.format(name))
        print_('ext: {}'.format(ext))
        print_('_datetime: {}'.format(_datetime))
        raise e

    return Image(url=url, filename='[{}] {}{}'.format(_datetime.strftime("%Y-%m-%d"), name, ext))


def get_imgs(user, title, session, cw=None):
    print_ = get_print(cw)

    imgs = []
    url = 'https://{}.newgrounds.com/art'.format(user)
    params = {'page': 1, 'isAjaxRequest': 1}
    while check_alive(cw):
        posts = get_posts(url, params, session, print_)

        if not posts:
            break

        for post in posts:
            sleep(0.75)
            imgs.append(get_img(post, session, print_))
        s = '{} {}  -  {}'.format(tr_('읽는 중...'), title, len(imgs))
        if cw:
            cw.setTitle(s)
        print_('processed: {}'.format(len(imgs)))
        print_('page: {}'.format(params['page']))
        params['page'] += 1

    return imgs



File: src/extractor/xnxx_downloader.py

import downloader
from utils import Soup, Downloader, LazyUrl, format_filename
import ree as re
from m3u8_tools import playlist2stream
from io import BytesIO as IO



class Video:

    def __init__(self, url, url_page, title, url_thumb):
        self._url = url
        self.url = LazyUrl(url_page, self.get, self)
        self.id = get_id(url_page)
        self.title = title
        self.filename = format_filename(title, self.id, '.mp4')
        f = IO()
        self.url_thumb = url_thumb
        downloader.download(url_thumb, buffer=f)
        self.thumb = f

    def get(self, _):
        return self._url


def get_id(url):
    return url.split('xnxx.com/')[1].split('/')[0]



class Downloader_xnxx(Downloader):
    type = 'xnxx'
    URLS = [r'regex:xnxx[0-9]*\.(com|es)']
    single = True
    display_name = 'XNXX'
    ACCEPT_COOKIES = [r'(.*\.)?xnxx[0-9]*\.(com|es)']

    @classmethod
    def fix_url(cls, url):
        return re.sub(r'xnxx[0-9]*\.(com|es)', 'xnxx.com', url)

    def read(self):
        video = get_video(self.url)
        self.urls.append(video.url)
        self.setIcon(video.thumb)
        self.title = video.title


def get_video(url):
    html = downloader.read_html(url)
    soup = Soup(html)

    for script in soup.findAll('script'):
        script = script.text or script.string or ''
        hls = re.find(r'''html5player\.setVideoHLS\(['"](.+?)['"]''', script)
        if hls:
            break
    else:
        raise Exception('No VideoHLS')

    video = playlist2stream(hls)

    title = get_title(soup)

    url_thumb = soup.find('meta', {'property': 'og:image'}).attrs['content'].strip()

    video = Video(video, url, title, url_thumb)
    return video


def get_title(soup):
    return soup.find('meta', {'property': 'og:title'}).attrs['content'].strip()



File: src/extractor/baraag_downloader.py

#coding:utf8
from utils import Downloader, clean_title, Session
from mastodon import get_info
import ree as re



def get_id(url):
    return re.find('baraag.net/([^/]+)', url.lower())



class Downloader_baraag(Downloader):
    type = 'baraag'
    URLS = ['baraag.net']
    display_name = 'baraag.net'
    ACCEPT_COOKIES = [r'(.*\.)?baraag\.net']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        id_ = get_id(url) or url
        return f'https://baraag.net/{id_}'

    def read(self):
        id_ = get_id(self.url)
        info = get_info('baraag.net', id_, f'baraag_{id_}', self.session, self.cw)

        self.urls += info['files']

        self.title = clean_title('{} (baraag_{})'.format(info['title'], id_))



File: src/extractor/navertv_downloader.py

import downloader
import ree as re
from io import BytesIO as IO
from utils import Downloader, LazyUrl, get_ext, format_filename, try_n
import ytdl



class Downloader_navertv(Downloader):
    type = 'navertv'
    single = True
    URLS = ['tv.naver.com']
    display_name = 'Naver TV'
    ACCEPT_COOKIES = [r'(.*\.)?naver\.com']

    @classmethod
    def fix_url(cls, url):
        if not re.match(r'https?://.+', url, re.I):
            url = f'https://tv.naver.com/v/{url}'
        return url

    def read(self):
        video = Video(self.url, cw=self.cw)
        video.url()#

        self.urls.append(video.url)
        self.setIcon(video.thumb)

        self.enableSegment()

        self.title = video.title



class Video:
    _url = None

    def __init__(self, url, cw=None):
        self.url = LazyUrl(url, self.get, self)
        self.cw = cw

    @try_n(4)
    def get(self, url):
        if self._url:
            return self._url

        ydl = ytdl.YoutubeDL(cw=self.cw)
        info = ydl.extract_info(url)
        fs = [f for f in info['formats'] if f['protocol'] in ['http', 'https']]
        fs = sorted(fs, key=lambda f: int(f.get('width', 0)), reverse=True)
        if not fs:
            raise Exception('No MP4 videos')
        f = fs[0]
        self._url = f['url']

        self.thumb_url = info['thumbnails'][0]['url']
        self.thumb = IO()
        downloader.download(self.thumb_url, buffer=self.thumb)
        self.title = info['title']
        id = info['id']
        ext = get_ext(self._url)
        self.filename = format_filename(self.title, id, ext)
        return self._url



File: src/extractor/lhscan_downloader.py

#coding:utf8
import downloader
from utils import Soup, urljoin, LazyUrl, Downloader, try_n, Session, clean_title, get_print, check_alive
import os
from translator import tr_
import page_selector
import clf2
import utils
import base64
import ree as re
import errors
##from image_reader import QPixmap


class Image:
    def __init__(self, url, page, p):
        self._url = url
        self.url = LazyUrl(page.url, self.get, self)#, pp=self.pp)
        ext = os.path.splitext(url)[1]
        if ext.lower()[1:] not in ['jpg', 'jpeg', 'bmp', 'png', 'gif', 'webm', 'webp']:
            ext = '.jpg'
        self.filename = '{}/{:04}{}'.format(page.title, p, ext)

    def get(self, _):
        return self._url

##    def pp(self, filename):
##        pixmap = QPixmap(filename)
##        pixmap.save(filename)
##        return filename


class Page:
    def __init__(self, title, url):
        self.title = clean_title(title)
        self.url = url


def get_soup_session(url, cw=None, win=None):
    print_ = get_print(cw)
    session = Session()
    res = clf2.solve(url, session=session, cw=cw, win=win)
    print_('{} -> {}'.format(url, res['url']))
    if res['url'].rstrip('/') == 'https://welovemanga.one':
        raise errors.LoginRequired()
    return Soup(res['html']), session



class Downloader_lhscan(Downloader):
    type = 'lhscan'
    URLS = [
        #'lhscan.net', 'loveheaven.net',
        'lovehug.net', 'welovemanga.', 'nicomanga.com',
        ]
    MAX_CORE = 16
    display_name = 'LHScan'
    ACCEPT_COOKIES = [rf'(.*\.)?{domain}' for domain in URLS]

    def init(self):
        self.soup, self.session = get_soup_session(self.url, self.cw)
        try:
            self.name
        except:
            raise errors.Invalid('{}: {}'.format(tr_('목록 주소를 입력해주세요'), self.url))

    @classmethod
    def fix_url(cls, url):
        url = url.replace('lovehug.net', 'welovemanga.one')
        url = url.replace('welovemanga.net', 'welovemanga.one') #4298
        return url

    @property
    def name(self):
        title = self.soup.find('ul', class_='manga-info').find('h3').text
        return clean_title(title)

    def read(self):
        self.title = tr_('읽는 중... {}').format(self.name)

        imgs = get_imgs(self.url, self.name, self.session, self.soup, self.cw)

        for img in imgs:
            self.urls.append(img.url)

        self.title = self.name


@try_n(8)
def get_imgs_page(page, referer, session, cw=None):
    print_ = get_print(cw)
    print_(page.title)

    html = downloader.read_html(page.url, referer, session=session)
    if clf2._is_captcha(Soup(html)): #4124
        html = clf2.solve(page.url, session, cw)['html']
    if not html:
        raise Exception('empty html')
    try:
        html = html.replace('{}='.format(re.find(r"\$\(this\)\.attr\('(.+?)'", html, err='no cn')), 'data-src=')
    except: #5351
        pass
    soup = Soup(html)

    m = re.find(r'''(load_image|imgsListchap)\(([0-9]+)''', html)
    if m: #6186
        cid = m[1]
        if utils.domain(page.url, 2).lower() == 'nicomanga.com':
            url_api = urljoin(page.url, f'/app/manga/controllers/cont.imgsList.php?cid={cid}')
        else:
            url_api = urljoin(page.url, f'/app/manga/controllers/cont.listImg.php?cid={cid}')
        soup = downloader.read_soup(url_api, page.url, session=session)

    imgs = []
    for img in soup.findAll('img', class_='chapter-img'):
        src = img.get('data-pagespeed-lazy-src') or img.get('data-src') or img.get('data-srcset') or img.get('data-aload') or img.get('data-original') or img['src']
        try:
            src = base64.b64decode(src).strip().decode('utf8')
        except:
            pass
        src0 = src
        src = src.replace('welovemanga.one', '1')#
        src = urljoin(page.url, src).strip()
        if 'Credit_LHScan_' in src or '5e1ad960d67b2_5e1ad962338c7' in src:
            continue
        if 'fe132b3d32acc39f5adcea9075bedad4LoveHeaven' in src:
            continue
        if 'LoveHug_600cfd96e98ff.jpg' in src:
            continue
        if 'image_5f0ecf23aed2e.png' in src:
            continue
        if '/uploads/lazy_loading.gif' in src:
            continue
        if '/xstaff.jpg.pagespeed.ic.gPQ2SGcYaN.webp' in src:
            continue
        if '/uploads/loading-mm.gif' in src:
            continue
        src = src.replace('\n', '').replace('\r', '') #5238
        #6105
##        if 'proxy.php?link=' not in src: #5351
##            src = 'https://welovekai.com/proxy.php?link=' + src #5238
        if not imgs:
            print_(src0)
            print_(src)
        img = Image(src, page, len(imgs))
        imgs.append(img)

    return imgs


def get_pages(url, session, soup=None, cw=None):
    if soup is None:
        html = downloader.read_html(url, session=session)
        soup = Soup(html)

    tab = soup.find('ul', class_='list-chapters')

    pages = []
    for li in tab.findAll('li'):
        text = li.find('div', class_='chapter-name').text.strip()
        href = li.parent['href']
        href = urljoin(url, href)
        page = Page(text, href)
        pages.append(page)

    if not pages:
        raise Exception('no pages')

    return pages[::-1]


@page_selector.register('lhscan')
def f(url, win):
    soup, session = get_soup_session(url, win=win)
    pages = get_pages(url, session, soup=soup)
    return pages


@try_n(2)
def get_imgs(url, title, session, soup=None, cw=None):
    if soup is None:
        html = downloader.read_html(url, session=session)
        soup = Soup(html)

    pages = get_pages(url, session, soup, cw)
    pages = page_selector.filter(pages, cw)

    imgs = []
    for i, page in enumerate(pages):
        check_alive(cw)
        imgs += get_imgs_page(page, url, session, cw)
        s = '{} {} / {}  ({} / {})'.format(tr_('읽는 중...'), title, page.title, i+1, len(pages))
        if cw is not None:
            cw.setTitle(s)
        else:
            print(s)

    return imgs



File: src/extractor/tiktok_downloader.py

import downloader
import ree as re
from utils import Soup, LazyUrl, Downloader, try_n, compatstr, get_print, Session, get_max_range, format_filename, json
import clf2
import ytdl
from urllib.parse import unquote
PATTERN_VID = '/(v|video)/(?P<id>[0-9]+)'
SHOW = True


def is_captcha(soup, cw=None):
    r = soup.find('div', class_="verify-wrap") or soup.find('div', class_='captcha_verify_container')
    if r:
        get_print(cw)('captcha')
    return r



class Downloader_tiktok(Downloader):
    type = 'tiktok'
    single = True
    URLS = ['tiktok.com', 'douyin.com']
    display_name = 'TikTok'
    ACCEPT_COOKIES = [r'(.*\.)?(tiktok|douyin)\.com']

    def init(self):
        cw = self.cw
        self.session = Session()
        res = clf2.solve(self.url, self.session, cw)
        soup = Soup(res['html'])
        if is_captcha(soup, cw):
            def f(html):
                return not is_captcha(Soup(html))
            res = clf2.solve(self.url, self.session, cw, show=True, f=f)
        self.url = self.fix_url(res['url']) #4324

    @classmethod
    def fix_url(cls, url):
        url = url.split('?')[0].split('#')[0].strip('/')
        if '://' not in url:
            url = 'https://www.tiktok.com/@{}'.format(url)
        return url

    def read(self):
        format = compatstr(self.ui_setting.youtubeFormat.currentText()).lower().strip()

        def parse_video_url(info, item):
            if 'url' in item:
                return item['url']
            if 'tiktok.com' in self.url.lower(): # TikTok
                return 'https://www.tiktok.com/@{}/video/{}'.format(info.get('uid', ''), item['id']) #5235
            else: # Douyin
                return 'https://www.douyin.com/video/{}'.format(item['id'])

        if re.search(PATTERN_VID, self.url): # single video
            video = Video(self.url, self.session, format, self.cw)
            video.url()
            self.urls.append(video.url)
            self.title = video.title
        elif 'tiktok.com/tag/' in self.url or 'douyin.com/search/' in self.url: # tag search
            tag = re.find(r'/(tag|search)/([^/#\?]+)', self.url)[1]
            tag = unquote(tag)
            title = '#{}'.format(tag)
            info = read_channel(self.url, self.session, self.cw, title=title)
            items = info['items']
            videos = [Video(parse_video_url(info, item), self.session, format, self.cw) for item in items]
            video = self.process_playlist(title, videos)
        elif 'tiktok.com/@' in self.url or 'douyin.com/user/' in self.url: # channel
            info = read_channel(self.url, self.session, self.cw)
            items = info['items']
            videos = [Video(parse_video_url(info, item), self.session, format, self.cw) for item in items]
            title = '{} (tiktok_{})'.format(info['nickname'], info['uid'])
            video = self.process_playlist(title, videos)
        else:
            raise NotImplementedError()


class Video:
    _url = None

    def __init__(self, url, session, format, cw):
        self.url = LazyUrl(url, self.get, self)
        self.session = session
        self.format = format
        self.cw = cw

    @try_n(2)
    def get(self, url):
        if self._url:
            return self._url
        m = re.search(PATTERN_VID, url)
        id = m.group('id')

        ydl = ytdl.YoutubeDL(cw=self.cw)
        info = ydl.extract_info(url)

        ext = '.mp4'
        self.title = info['title']
        self.filename = format_filename(self.title, id, ext)

        self._url = info['url']

        return self._url


def read_channel(url, session, cw=None, title=None):
    info = {}
    info['items'] = []

    if 'tiktok.com' in url.lower(): # TikTok
        soup = downloader.read_soup(url, session=session, user_agent='facebookexternalhit/1.1')
        info['uid'] = re.find(r'/@([\w\.-]+)', soup.find('meta', {'property': 'og:url'})['content'], err='no uid')
        nick = soup.find('meta', {'property': 'og:title'})['content']
        if nick.endswith(' on TikTok'):
            nick = nick[:-len(' on TikTok')]
        info['nickname'] = nick
    else: # Douyin
        def f(html, browser=None):
            soup = Soup(html)
            if is_captcha(soup):
                browser.show()
                return False
            try:
                info['uid'] = re.find(r'''uniqueId%22%3A%22(.+?)%22''', html, err='no uid')
                info['nickname'] = json.loads(re.findall(r'''"name"\s*:\s*(".+?")''', html)[-1]) #5896
                return True
            except:
                return False
        clf2.solve(url, session, cw, f=f)

    options = {
        'extract_flat': True,
        'playlistend': get_max_range(cw),
        }
    ydl = ytdl.YoutubeDL(options, cw=cw)
    info_ = ydl.extract_info(url)

    for e in info_['entries']:
        info['items'].append({'url': e['webpage_url']})

    if not info['items']:
        raise Exception('no items')

    return info




File: src/extractor/hanime_downloader.py

import downloader
from utils import Session, Downloader, try_n, Soup, format_filename, get_print, get_resolution, json
import ree as re
from io import BytesIO
import os
from timee import time
from m3u8_tools import M3u8_stream
from random import randrange


class Video:

    def __init__(self, info, stream):
        self.info = info
        self.id = info['id']
        self.title = info['name']
        self.brand = info['brand']
        self.url = stream['url']
        self.url_thumb = info['poster_url']
        self.thumb = BytesIO()
        downloader.download(self.url_thumb, buffer=self.thumb)
        ext = os.path.splitext(self.url.split('?')[0].split('#')[0])[1]
        if ext.lower() == '.m3u8':
            ext = '.mp4'
            self.url = M3u8_stream(self.url, n_thread=4)
            for i, seg in self.url.urls[-20:]:
                seg._ignore_err = True #5272
        else:
            size = downloader.get_size(self.url)
            if size <= 0:
                raise Exception('Size is 0')
        self.filename = format_filename('[{}] {}'.format(self.brand, self.title), self.id, ext)

    def __repr__(self):
        return f'Video({self.id})'



class Downloader_hanime(Downloader):
    type = 'hanime'
    URLS = ['hanime.tv/hentai-videos/', 'hanime.tv/videos/']
    single = True
    display_name = 'hanime.tv'
    ACCEPT_COOKIES = [r'(.*\.)?hanime\.tv']

    def init(self):
        self.session = Session('chrome')

    def read(self):
        video = get_video(self.url, self.session, cw=self.cw)
        self.video = video

        self.urls.append(video.url)
        self.filenames[video.url] = video.filename

        self.setIcon(video.thumb)
        self.title = '[{}] {}'.format(video.brand, video.title)


@try_n(8)
def get_video(url, session, cw=None):
    print_ = get_print(cw)
    session.headers['X-Directive'] = 'api'
    html = downloader.read_html(url, session=session)
    soup = Soup(html)
    for script in soup.findAll('script'):
        script = script.text or script.string or ''
        data = re.find('window.__NUXT__=(.+)', script)
        if data is not None:
            data = data.strip()
            if data.endswith(';'):
                data = data[:-1]
            data = json.loads(data)
            break
    else:
        raise Exception('No __NUXT__')

    info = data['state']['data']['video']['hentai_video']
    query = info['slug']
    #url_api = 'https://members.hanime.tv/api/v3/videos_manifests/{}?'.format(query) # old
    url_api = 'https://hanime.tv/rapi/v7/videos_manifests/{}?'.format(query) # new
    hdr = {
        'x-signature': ''.join('{:x}'.format(randrange(16)) for i in range(32)),
        'x-signature-version': 'web2',
        'x-time': str(int(time())),
        }
    r = session.get(url_api, headers=hdr)
    data = json.loads(r.text)
    streams = []
    for server in data['videos_manifest']['servers']:
        streams += server['streams']

    streams_good = []
    for stream in streams:
        url_video = stream['url']
        if not url_video or 'deprecated.' in url_video:
            continue
        stream['height'] = int(stream['height'])
        streams_good.append(stream)

    if not streams_good:
        raise Exception('No video available')
    res = get_resolution()

    def print_stream(stream):
        print_([stream['extension'], stream['height'], stream['filesize_mbs'], stream['url']])

    steams_filtered = []
    for stream in streams_good:
        print_stream(stream)
        if stream['height'] <= res: #3712
            steams_filtered.append(stream)

    if steams_filtered:
        stream = sorted(steams_filtered, key=lambda _: _['height'])[-1]
    else:
        stream = sorted(streams_good, key=lambda _: _['height'])[0]

    print_('Final stream:')
    print_stream(stream)
    return Video(info, stream)



File: src/extractor/kissjav_downloader.py

import downloader
from utils import urljoin, Downloader, LazyUrl, Session, try_n, format_filename, get_resolution, get_print
import ree as re
from io import BytesIO
import clf2



class Downloader_kissjav(Downloader):
    type = 'kissjav'
    URLS = ['kissjav.com', 'kissjav.li', 'mrjav.net'] #4835
    single = True
    display_name = 'KissJAV'
    ACCEPT_COOKIES = [r'(.*\.)?(kissjav|mrjav)\.(com|li|net)']

    def read(self):
        self.session = Session()#get_session(self.url, cw=self.cw)

        video = get_video(self.url, self.session, self.cw)
        self.urls.append(video.url)
        self.setIcon(video.thumb)
        self.enableSegment(1024*1024//2)

        self.title = video.title


@try_n(2)
def get_video(url, session, cw):
    print_ = get_print(cw)
    soup = downloader.read_soup(url, session=session)

    view = soup.find('div', id='player-container-fluid')
    fs = []
    for source in view.findAll('source'):
        src = urljoin(url, source.attrs['src'])
        res = re.find('([0-9]+)p', source.attrs['title'])
        res = int(res) if res else 0
        f = {'res': res, 'src': src}
        fs.append(f)
        print_(f)

    if not fs:
        raise Exception('No source')

    #4773
    res = max(get_resolution(), min(f['res'] for f in fs))
    print_(f'res: {res}')
    fs = sorted([f for f in fs if f['res'] <= res], key=lambda f: f['res'])
    f = fs[-1]
    print_(f'best: {f}')
    src_best = f['src']

    title = soup.find('h1').text.strip()
    id = soup.find('div', id='video').attrs['data-id']

    url_thumb = soup.find('meta', {'property': 'og:image'}).attrs['content']

    #src_best = downloader.real_url(src_best)

    video = Video(src_best, url_thumb, url, title, id, session)
    return video


class Video:
    def __init__(self, url, url_thumb, referer, title, id, session):
        self.title = title
        self.filename = format_filename(title, id, '.mp4')
        self.url = LazyUrl(referer, lambda x: url, self)

        self.thumb = BytesIO()
        self.url_thumb = url_thumb
        downloader.download(url_thumb, buffer=self.thumb, session=session)


@try_n(2)
def get_session(url, cw=None):
    session = Session()
    clf2.solve(url, session=session, cw=cw)
    return session



File: src/extractor/kakuyomu_downloader.py

#coding:utf8
import downloader
import utils
from utils import Soup, urljoin, Downloader, LazyUrl, try_n, clean_title, get_print, json, File
import os
from io import BytesIO
from translator import tr_



class Page(File):
    type = 'kakuyomu'
    format = 'title'

    def __init__(self, info):
        info['title_all'] = clean_title('[{:04}] {}'.format(info['p'], info['title']))
        d = {
            'title': info['title_all'],
            }
        info['name'] = utils.format(self.type, d, '.txt')
        super().__init__(info)

    def get(self):
        text = get_text(self)
        f = BytesIO()
        f.write(text.encode('utf8'))
        f.seek(0)
        return {'url': f}



class Downloader_kakuyomu(Downloader):
    type = 'kakuyomu'
    URLS = ['kakuyomu.jp']
    MAX_CORE = 2
    detect_removed = False
    display_name = 'カクヨム'
    ACCEPT_COOKIES = [r'(.*\.)?kakuyomu\.jp']
    atts = ['info_title', 'info_description']

    def read(self):
        self.info = get_info(self.url, cw=self.cw)
        self.artist = self.info['artist']
        title_dir = clean_title('[{}] {}'.format(self.artist, self.info['title']))

        outdir = utils.dir(self.type, title_dir, self.cw)

        self.urls += self.info['pages']

        self.title = title_dir
        self.info_title = self.info['title']
        self.info_description = self.info['description']

    def post_processing(self):
        names = self.cw.names
        filename = clean_title('[merged] [{}] {}'.format(self.artist, self.info_title), n=-4) + '.txt'
        filename = os.path.join(self.dir, filename)
        try:
            with utils.open(filename, 'wb') as f:
                f.write('    {}\n\n    作者：{}\n\n\n'.format(self.info_title, self.artist).encode('utf8'))
                f.write(self.info_description.encode('utf8'))
                for i, file in enumerate(names):
                    self.cw.pbar.setFormat('[%v/%m]  {} [{}/{}]'.format(tr_('병합...'), i, len(names)))
                    with open(file, 'rb') as f_:
                        text = f_.read()
                    f.write(b'\n\n\n\n')
                    f.write(text)
        finally:
            self.cw.pbar.setFormat('[%v/%m]')


@try_n(4, sleep=30)
def get_text(page):
    html = downloader.read_html(page['referer'])
    soup = Soup(html)
    view = soup.find('div', class_='widget-episodeBody')
    story = view.text.strip()
    text = '''────────────────────────────────

  ◆  {}        {}

────────────────────────────────


{}'''.format(page['title_all'], page['date'], story)
    return text


def get_info(url, soup=None, cw=None):
    print_ = get_print(cw)
    if soup is None:
        html = downloader.read_html(url)
        soup = Soup(html)

    info = {}

    rdata = soup.find('script', id='__NEXT_DATA__').string #6620
    data = json.loads(rdata)

    wid = data['query']['workId']
    info['title'] = data['props']['pageProps']['__APOLLO_STATE__'][f'Work:{wid}']['title']
    aid = data['props']['pageProps']['__APOLLO_STATE__'][f'Work:{wid}']['author']['__ref']
    info['artist'] = data['props']['pageProps']['__APOLLO_STATE__'][f'{aid}']['activityName']

    catch = data['props']['pageProps']['__APOLLO_STATE__'][f'Work:{wid}'].get('catchphrase') or ''
    intro = data['props']['pageProps']['__APOLLO_STATE__'][f'Work:{wid}'].get('introduction') or ''
    desc = '  {}{}'.format(catch, ('\n\n\n'+intro) if intro else '')
    info['description'] = desc

    eps = []
    for tc in  data['props']['pageProps']['__APOLLO_STATE__'][f'Work:{wid}']['tableOfContents']:
        _ = data['props']['pageProps']['__APOLLO_STATE__'][tc['__ref']].get('episodes')
        if _:
            eps += _
        else: #6708
            eps += data['props']['pageProps']['__APOLLO_STATE__'][tc['__ref']]['episodeUnions']

    pages = []
    for ep in eps:
        eid = ep['__ref'].split('Episode:')[1]
        href = urljoin(url, f'/works/{wid}/episodes/{eid}')
        subtitle = data['props']['pageProps']['__APOLLO_STATE__'][ep['__ref']]['title']
        date = data['props']['pageProps']['__APOLLO_STATE__'][ep['__ref']]['publishedAt']
        page = Page({'referer': href, 'title': subtitle, 'date': date, 'p': len(pages)+1})
        pages.append(page)

    info['pages'] = pages

    return info



File: src/extractor/mastodon_downloader.py

#coding:utf8
from utils import Downloader, clean_title, Session
from mastodon import get_info
import ree as re



def get_id(url):
    return re.find('mastodon.social/([^/]+)', url.lower())



class Downloader_mastodon(Downloader):
    type = 'mastodon'
    URLS = ['mastodon.social']
    ACCEPT_COOKIES = [r'(.*\.)?mastodon\.social']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        id_ = get_id(url) or url
        return f'https://mastodon.social/{id_}'

    def read(self):
        id_ = get_id(self.url)
        info = get_info('mastodon.social', id_, f'mastodon_{id_}', self.session, self.cw)

        self.urls += info['files']

        self.title = clean_title('{} (mastodon_{})'.format(info['title'], id_))



File: src/extractor/twitch_downloader.py

#coding: utf8
import downloader
import ytdl
from utils import Downloader, LazyUrl, try_n, format_filename, get_ext, Session, get_print, get_resolution, get_max_range, print_error, json
from io import BytesIO
from m3u8_tools import M3u8_stream
import ree as re
import errors
import utils
import os



class Downloader_twitch(Downloader):
    type = 'twitch'
    URLS = ['twitch.tv']
    single = True
    ACCEPT_COOKIES = [r'.*(twitch|ttvnw|jtvnw).*']

    def init(self):
        url = self.url
        if 'twitch.tv' in url:
            if not url.startswith('http://') and not url.startswith('https://'):
                url = 'https://' + url
            self.url = url
        else:
            url = f'https://www.twitch.tv/videos/{url}'
            self.url = url
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        url = url.replace('m.twitch.tv', 'www.twitch.tv')
        if re.search(r'/(videos|clips)\?filter=', url):
            return url.strip('/')
        url = url.split('?')[0].strip('/')
        filter = cls.get_filter(url)
        if filter == 'live':
            url = '/'.join(url.split('/')[:4])
        return url

    @classmethod
    def get_filter(cls, url):
        if url.count('/') == 3:
            if 'www.twitch.tv' in url or '//twitch.tv' in url:
                filter = 'live'
            else:
                filter = None
        elif url.count('/') == 4:
            filter = re.find(r'filter=([0-9a-zA-Z_]+)', url) or re.find(r'[0-9a-zA-Z_]+', url.split('/')[-1])
            if filter is not None and filter.isdigit():
                filter = None
        else:
            filter = None
        if filter in ['about', 'schedule']:
            filter = 'live'
        return filter

    def read(self):
        if '/directory/' in self.url.lower():
            raise errors.Invalid(f'[twitch] Directory is unsupported: {self.url}')

        filter = self.get_filter(self.url)

        if filter is None:
            video = Video(self.url, self.session, self.cw)
            video.url()
            self.urls.append(video.url)
            self.title = video.title
        elif filter == 'live':
            video = Video(self.url, self.session, self.cw, live=True)
            video.url()
            self.urls.append(video.url)
            self.title = os.path.splitext(video.filename)[0].replace('：', ':')
        elif filter == 'clips':
            info = get_videos(self.url, cw=self.cw)
            video = self.process_playlist('[Clip] {}'.format(info['name']), info['videos'])
        else:
            raise NotImplementedError(filter)

        self.artist = video.artist

        thumb = BytesIO()
        downloader.download(video.url_thumb, buffer=thumb) #5418

        self.setIcon(thumb)
        if filter == 'live':
            d = {}
            d['url'] = self.url
            d['title'] = self.artist
            d['thumb'] = thumb.getvalue()
            utils.update_live(d, self.cw)


@try_n(2)
def get_videos(url, cw=None):
    print_ = get_print(cw)
    print_(f'get_videos: {url}')
    info = {}
    options = {
            'extract_flat': True,
            'playlistend': get_max_range(cw),
            }
    videos = []
    ydl = ytdl.YoutubeDL(options, cw=cw)
    info = ydl.extract_info(url)
    for e in info['entries']:
        video = Video(e['url'], self.session, cw)
        video.id = int(e['id'])
        videos.append(video)
        if 'name' not in info:
            info['name'] = ydl.extract_info(e['url'])['creator']
    if not videos:
        raise Exception('no videos')
    info['videos'] = sorted(videos, key=lambda video: video.id, reverse=True)
    return info


def alter(seg, cw):
    if 'amazon' in seg.raw.title.lower():
        get_print(cw)('strip ads')
        return []
    segs = []
    if '-muted' in seg.url:
        seg_ = seg.copy()
        seg_.url = seg.url.replace('-muted', '')
        segs.append(seg_)
    segs.append(seg)
    return segs


def extract_info(url, cw=None):
    print_ = get_print(cw)
    ydl = ytdl.YoutubeDL(cw=cw)
    try:
        info = ydl.extract_info(url)
    except Exception as e:
        ex = type(ytdl.get_extractor(url))(ydl)
        _download_info = getattr(ex, '_download_info', None)
        if _download_info is not None:
            vod_id = ex._match_id(url)
            info = _download_info(vod_id)
            print_(info)
        if 'HTTPError 403' in str(e):
            raise errors.LoginRequired()
        raise
    return info


class Video:
    _url = None

    def __init__(self, url, session, cw, live=False):
        self.url = LazyUrl(url, self.get, self)
        self.session = session
        self.cw = cw
        self._live = live

    @try_n(4)
    def get(self, url):
        print_ = get_print(self.cw)
        session = self.session
        if self._url:
            return self._url
        info = extract_info(url, self.cw)
        self.artist = info.get('creator') or info.get('uploader') #4953, #5031

        def print_video(video):
            #print_(video)#
            print_('{}[{}] [{}] [{}] {}'.format('LIVE ', video['format_id'], video.get('height'), video.get('tbr'), video['url']))

        videos = [video for video in info['formats'] if video.get('height')]

        videos = sorted(videos, key=lambda video:(video.get('height', 0), video.get('tbr', 0)), reverse=True)

        for video in videos:
            print_video(video)

        for video in videos:
            if video.get('height', 0) <= get_resolution(): #3723
                video_best = video
                break
        else:
            video_best = videos[-1]
        print_video(video)

        video = video_best['url']

        ext = get_ext(video)
        id = info['display_id']

        if self._live:
            self.title = info['description']
            if utils.SD['twitch']['strip_ads']:
                video = M3u8_stream(video, n_thread=4, alter=alter, session=session)
            else:
                video = utils.LiveStream(video, headers=video_best.get('http_headers', {}))
            ext = '.mp4'
        else:
            self.title = info['title']
            if ext.lower() == '.m3u8':
                video = M3u8_stream(video, n_thread=4, alter=alter, session=session)
                ext = '.mp4'
        self.filename = format_filename(self.title, id, ext, artist=self.artist, live=self._live)
        self.url_thumb = info['thumbnail']
        self._url = video
        return self._url


def get_streamer_name(url):
    session = Session()
    session.purge('twitch')
    graphql_url = 'https://gql.twitch.tv/gql'
    headers = {
        'Client-ID': 'kimne78kx3ncx6brgo4mv6wki5h1ko',
        'Content-Type': 'application/json',
    }
    session.headers.update(headers)

    id = url.split('/')[3]

    payload = {'operationName': 'PlaybackAccessToken_Template', 'query': 'query PlaybackAccessToken_Template($login: String!, $isLive: Boolean!, $vodID: ID!, $isVod: Boolean!, $playerType: String!) {  streamPlaybackAccessToken(channelName: $login, params: {platform: "web", playerBackend: "mediaplayer", playerType: $playerType}) @include(if: $isLive) {    value    signature   authorization { isForbidden forbiddenReasonCode }   __typename  }  videoPlaybackAccessToken(id: $vodID, params: {platform: "web", playerBackend: "mediaplayer", playerType: $playerType}) @include(if: $isVod) {    value    signature   __typename  }}', 'variables': {'isLive': True, 'login': id, 'isVod': False, 'vodID': '', 'playerType': 'site'}}
    r = session.post(graphql_url, json=payload)
    r.raise_for_status()
    data = r.json()
    value = json.loads(data['data']['streamPlaybackAccessToken']['value'])
    cid = value['channel_id']
    utils.log(data)

    payload = [{"operationName":"EmotePicker_EmotePicker_UserSubscriptionProducts","variables":{"channelOwnerID":f"{cid}"},"extensions":{"persistedQuery":{"version":1,"sha256Hash":"71b5f829a4576d53b714c01d3176f192cbd0b14973eb1c3d0ee23d5d1b78fd7e"}}}]
    r = session.post(graphql_url, json=payload)
    r.raise_for_status()
    data = r.json()
    return data[0]['data']['user']['displayName']


class Live_twitch(utils.Live):
    type = 'twitch'

    @classmethod
    def is_live(cls, url):
        return Downloader.get('twitch').get_filter(url) == 'live'

    @classmethod
    def check_live(cls, url, info=None):
        if info is not None:
            try:
                info['title'] = get_streamer_name(url)
            except Exception as e:
                utils.log(print_error(e))
        ydl = ytdl.YoutubeDL(type='twitch')
        try:
            ydl.extract_info(url)
            return True
        except Exception as e:
            print(e)
            return False



File: src/extractor/fc2_downloader.py

import downloader
import ree as re
from utils import urljoin, Downloader, format_filename, Soup, LazyUrl, get_print, Session
from m3u8_tools import M3u8_stream
from io import BytesIO
PATTERN_ID = r'/content/([^/]+)'



class Downloader_fc2(Downloader):
    type = 'fc2'
    single = True
    URLS = ['video.fc2.com']
    ACCEPT_COOKIES = [r'(.*\.)?fc2\.com']

    @classmethod
    def fix_url(cls, url):
        if not re.match(r'https?://.+', url, re.I):
            url = f'https://video.fc2.com/content/{url}'
        return url

    @classmethod
    def key_id(cls, url):
        return re.find(PATTERN_ID, url) or url

    def read(self):
        self.session = Session()
        self.session.cookies.set('_ac', '1', domain='.video.fc2.com')
        info = get_info(self.url, self.session, self.cw)

        video = info['videos'][0]

        self.urls.append(video.url)

        f = BytesIO()
        downloader.download(video.url_thumb, referer=self.url, buffer=f)
        self.setIcon(f)

        self.title = info['title']


class Video:

    def __init__(self, url, url_thumb, referer, title, id_, session):
        self._url = url
        self.url = LazyUrl(referer, self.get, self)
        self.filename = format_filename(title, id_, '.mp4')
        self.url_thumb = url_thumb
        self.session = session

    def get(self, referer):
        ext = downloader.get_ext(self._url, session=self.session, referer=referer)
        if ext == '.m3u8':
            video = M3u8_stream(self._url, referer=referer, session=self.session, n_thread=4)
        else:
            video = self._url
        return video


def get_info(url, session, cw=None):
    print_ = get_print(cw)
    info = {'videos': []}
    html = downloader.read_html(url, session=session)
    soup = Soup(html)
    info['title'] = soup.find('h2', class_='videoCnt_title').text.strip()

    id_ = re.find(PATTERN_ID, url, err='no id')
    print_('id: {}'.format(id_))
    token = re.find(r'''window.FC2VideoObject.push\(\[['"]ae['"], *['"](.+?)['"]''', html, err='no token')
    print_('token: {}'.format(token))

    url_api = 'https://video.fc2.com/api/v3/videoplaylist/{}?sh=1&fs=0'.format(id_)
    hdr = {
        'X-FC2-Video-Access-Token': token,
        }
    data = downloader.read_json(url_api, url, session=session, headers=hdr)

    pl = data['playlist']
    url_video = urljoin(url, pl.get('hq') or pl.get('nq') or pl['sample']) #3784
    url_thumb = soup.find('meta', {'property':'og:image'})['content']
    video = Video(url_video, url_thumb, url, info['title'], id_, session)
    info['videos'].append(video)

    return info



File: src/extractor/flickr_downloader.py

from utils import Downloader, File, Session, urljoin, get_ext, clean_title, Soup, limits
import utils
import ree as re
import downloader
import clf2
from timee import time
TIMEOUT = 10


class File_flickr(File):
    type = 'flickr'
    format = '[date] id'

    @limits(1)
    def get(self):
        url = self['referer']
        soup = downloader.read_soup(url, session=self.session)
        img = soup.find('meta', {'property': 'og:image'})['content']
        date = re.find(r'"dateCreated":{"data":"([0-9]+)"', soup.html, err='no date')
        ext = get_ext(img)
        d = {
            'date': int(date),
            'id': re.find(r'/photos/[^/]+/([0-9]+)', url, err='no id'),
            }
        return {'url': img, 'name': utils.format('flickr', d, ext)}


class Downloader_flickr(Downloader):
    type = 'flickr'
    URLS = ['flickr.com']
    MAX_CORE = 4
    ACCEPT_COOKIES = [r'(.*\.)?flickr\.com']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        url = url.replace('flickr.com/people/', 'flickr.com/photos/')
        uid = re.find(r'flickr.com/photos/([^/]+)', url)
        if uid:
            url = f'https://www.flickr.com/photos/{uid}'
        return url

    def read(self):
        tab = ''.join(self.url.split('/')[3:4])
        if tab == 'photos':
            uid = self.url.split('/')[4]
            title = None
            ids = set()
            c = 0
            ct = None
            p_max = 1
            def f(html, browser=None):
                nonlocal title, c, ct, p_max
                soup = Soup(html)
                browser.runJavaScript('window.scrollTo(0,document.body.scrollHeight);')

                for a in soup.findAll('a'):
                    href = a.get('href') or ''
                    href = urljoin(self.url, href)
                    p_max = max(p_max, int(re.find(rf'flickr.com/photos/{uid}/page([0-9]+)', href) or 0))
                    id_ = re.find(rf'/photos/{uid}/([0-9]+)', href)
                    if not id_:
                        continue
                    if id_ in ids:
                        continue
                    ids.add(id_)
                    file = File_flickr({'referer': href})
                    self.urls.append(file)

                if ids:
                    uname = soup.h1.text.strip()
                    title = f'{clean_title(uname)} (flickr_{uid})'
                    self.cw.setTitle(f'{title} - {len(ids)}')
                    if c == len(ids):
                        if not ct:
                            ct = time()
                        dt = time() - ct
                        if dt > TIMEOUT:
                            return True
                    else:
                        ct = None
                    c = len(ids)

            p = 1
            while p <= p_max:
                url = f'https://www.flickr.com/photos/{uid}/page{p}'
                self.print_(url)
                clf2.solve(url, session=self.session, f=f)
                p += 1
            self.title = title
        else:
            raise NotImplementedError(tab)



File: src/extractor/v2ph_downloader.py

#coding:utf8
import downloader
from utils import get_ext, LazyUrl, Downloader, try_n, clean_title, get_print, print_error, limits
import ree as re
from translator import tr_
import errors
import clf2


def setPage(url, p):
    url = url.split('?')[0]
    if p > 1:
        url += '?page={}'.format(p)
    return url


def getPage(url):
    p = re.find('page=([0-9]+)', url)
    return int(p or 1)


class Image:
    def __init__(self, url, referer, p):
        self._url = url
        self.url = LazyUrl(referer, self.get, self)
        ext = get_ext(url)
        self.filename = '{:04}{}'.format(p, ext)

    @limits(.25)
    def get(self, _):
        return self._url



class Downloader_v2ph(Downloader):
    type = 'v2ph'
    URLS = ['v2ph.com/album/']
    MAX_CORE = 4
    MAX_PARALLEL = 1
    display_name = 'V2PH'
    ACCEPT_COOKIES = [r'(.*\.)?v2ph\.com']

    def init(self):
        self.session = clf2.solve(self.url)['session']

    @classmethod
    def fix_url(cls, url):
        return url.split('?')[0]

    def read(self):
        info = get_info(self.url, self.session)

        for img in get_imgs(self.url, self.session, info['title'], self.cw):
            self.urls.append(img.url)

        self.title = clean_title(info['title'])



@try_n(2)
def get_info(url, session):
    soup = read_soup(url, session)
    info = {}
    info['title'] = soup.find('h1').text.strip()
    return info


@try_n(4)
@limits(5)
def read_soup(url, session):
    return downloader.read_soup(url, session=session)


def get_imgs(url, session, title, cw=None):
    print_ = get_print(cw)
    imgs = []

    for p in range(1, 1001):
        url = setPage(url, p)
        print_(url)
        try:
            soup = read_soup(url, session)
        except Exception as e:
            if p > 1:
                print_(print_error(e))
                cw.showCookie() #6774
                cw.showLogin('https://www.v2ph.com/login', None, None)
                break
            else:
                raise e

        view = soup.find('div', class_='photos-list')
        if view is None:
            if p == 1:
                raise errors.LoginRequired()
            else:
                break # Guest user
        for img in view.findAll('img'):
            img = img.attrs['data-src']
            img = Image(img, url, len(imgs))
            imgs.append(img)

        pgn = soup.find('ul', class_='pagination')
        ps = [getPage(a.attrs['href']) for a in pgn.findAll('a')] if pgn else []
        if not ps or p >= max(ps):
            print('max p')
            break

        msg =  '{} {}  ({} / {})'.format(tr_('읽는 중...'), title, p, max(ps))
        if cw:
            cw.setTitle(msg)
        else:
            print(msg)

    return imgs



File: src/extractor/nozomi_downloader.py

import downloader
from urllib.parse import quote
from io import BytesIO
from utils import Downloader, query_url, get_ext, clean_title, check_alive, lock, get_print, get_max_range, File, Session, limits
import errors
from translator import tr_
import utils
import os



class File_nozomi(File):
    type = 'nozomi'
    format = 'idpage?'

    def get(self):
        infos = []
        for p, img in enumerate(read_post(self['id'], self['referer'], self.session, self.cw)):
            url = img['url']
            d = {
                'id': img['id'],
                'page?': f'_p{p}' if p else '',
                }
            filename = utils.format('nozomi', d, get_ext(url))
            info = {'url': url, 'name': filename, 'referer': img['referer']}
            infos.append(info)
        return infos


@limits(.25)
def read_post(id, referer, session, cw):
    print_ = get_print(cw)
    check_alive(cw)
    # https://j.nozomi.la/nozomi.js
    s_id = str(id)
    url_post = 'https://j.nozomi.la/post/{}/{}/{}.json'.format(s_id[-1], s_id[-3:-1], s_id)
    try:
        j = downloader.read_json(url_post, referer, session=session)
    except Exception as e:
        print_(f'{id}: {e}')
        return [] #5989
    imgs = []
    for url in j['imageurls']:
        did = url['dataid']
        if j.get('is_video'): #5754
            cdn = 'v'
            ext = url['type']
        else:
            cdn = 'g' if j.get('type') == 'gif' else 'w'
            ext = 'gif' if url.get('type') == 'gif' else 'webp'
        url = 'https://{}.nozomi.la/{}/{}/{}.{}'.format(cdn, did[-1], did[-3:-1], did, ext) #5340
        img = {'id': id, 'url': url, 'referer': f'https://nozomi.la/post/{id}.html'}
        imgs.append(img)
    return imgs


class Downloader_nozomi(Downloader):
    type = 'nozomi'
    URLS = ['nozomi.la']
    display_name = 'Nozomi.la'
    MAX_CORE = 15
    ACC_MTIME = True
    ACCEPT_COOKIES = [r'(.*\.)?nozomi\.la']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        return url.split('#')[0]

    @property
    def name(self):
        qs = query_url(self.url)
        name = qs['q'][0]
        if self._popular:
            name += ' - Popular'
        return clean_title(name)

    def read(self):
        if '/post/' in self.url:
            raise errors.Invalid(tr_('개별 다운로드는 지원하지 않습니다: {}').format(self.url))
        self._popular = 'search-Popular.' in self.url
        self.title = '{} {}'.format(tr_('읽는 중...'), self.name)
        qs = query_url(self.url)
        q = qs['q'][0]
        ids = get_ids_multi(q, self._popular, self.session, self.cw)
        self.print_(f'ids: {len(ids)}')
        max_pid = get_max_range(self.cw)

        def foo(id, p):
            d = {
                'id': id,
                'page?': f'_p{p}' if p else '',
                }
            filename_guess_base = utils.format('nozomi', d, '.webp')
            return os.path.join(utils.dir(self.type, self.name, self.cw), filename_guess_base)

        for id in ids:
            if os.path.isfile(foo(id, 0)):
                p = 0
                while True:
                    filename_guess = foo(id, p)
                    if not os.path.isfile(filename_guess):
                        break
                    self.urls.append(filename_guess)
                    p += 1
            else:
                file = File_nozomi({'id': id, 'url': f'https://nozomi.la/post/{id}.html', 'referer': self.url})
                self.urls.append(file)
            if len(self.urls) >= max_pid:
                break
        self.title = self.name


@lock
def get_ids(q, popular, session, cw):
    check_alive(cw)
    if q is None:
        if popular:
            url_api = 'https://j.nozomi.la/index-Popular.nozomi'
        else:
            url_api = 'https://j.nozomi.la/index.nozomi'
    else:
        q = q.replace('/', '') #5146
        if popular:
            url_api = 'https://j.nozomi.la/nozomi/popular/{}-Popular.nozomi'.format(quote(q))
        else:
            url_api = 'https://j.nozomi.la/nozomi/{}.nozomi'.format(quote(q))
    #print_(url_api)
    f = BytesIO()
    downloader.download(url_api, 'https://nozomi.la/', session=session, buffer=f)
    data = f.read()
    ids = []
    for i in range(0, len(data), 4):
        crop = data[i:i+4]
        id = crop[0]*16777216 + crop[1]*65536 + crop[2]*256 + crop[3]
        ids.append(id)
    return ids


def get_ids_multi(q, popular, session, cw=None):
    print_ = get_print(cw)
    max_pid = get_max_range(cw)
    qs = q.split(' ')
    qs_pos = [q for q in qs if not q.startswith('-')]
    qs_neg = [q[1:] for q in qs if q.startswith('-')]
    q = qs_pos[0] if qs_pos else None
    ids = get_ids(q, popular, session, cw)
    print_('{}: {}'.format(q, len(ids)))

    # Positive
    for q in qs_pos[1:]:
        ids_ = get_ids(q, popular, session, cw)
        set_ids_ = set(ids_)
        ids_old = ids
        ids = []
        for id in ids_old:
            if id in set_ids_:
                ids.append(id)
        print_('{}: {} ({})'.format(q, len(ids_), len(ids)))

    # Negative
    for q in qs_neg:
        ids_ = get_ids(q, popular, session, cw)
        set_ids_ = set(ids_)
        ids_old = ids
        ids = []
        for id in ids_old:
            if id not in set_ids_:
                ids.append(id)
        print_('-{}: {} ({})'.format(q, len(ids_), len(ids)))
    return ids[:max_pid]



File: src/extractor/discord_emoji_downloader.py

# coding: UTF-8
# title: Discord 서버 커스텀 이모지 다운로드
# author: SaidBySolo

"""
MIT License

Copyright (c) 2020 SaidBySolo

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from utils import Downloader, clean_title
import requests
import errors



class DownloaderDiscordEmoji(Downloader):
    type = "discord"

    def init(self):
        pass

    def read(self):
        token_guild_id_list = self.url.split(
            "/"
        )  # 값을 어떻게 받을지 몰라서 일단 나눴어요. discord_이메일/비밀번호/서버아이디 또는 discord_토큰/서버아이디 이런식으로 받게 해놨어요.

        if len(token_guild_id_list) == 2:
            token = token_guild_id_list[0]
            guild_id = token_guild_id_list[1]
        elif len(token_guild_id_list) == 3:
            email = token_guild_id_list[0]
            password = token_guild_id_list[1]
            guild_id = token_guild_id_list[2]

            response = self.post_account_info(email, password)
            account_info = response.json()
            if response.status_code == 400:
                if account_info.get("captcha_key"):
                    raise errors.Invalid(
                        "먼저 웹 또는 디스코드 앱에서 로그인하신후 캡차를 인증해주세요."
                    )  # 메세지 박스 return하니까 멈춰서 raise로 해놨어요
                else:
                    raise errors.Invalid("이메일 또는 비밀번호가 잘못되었습니다. 확인후 다시 시도해주세요.")
            else:
                if not account_info["token"]:
                    raise errors.Invalid("토큰을 받아오지 못했어요. 2단계인증을 사용중이신경우 토큰을 이용해 요청해주세요.")
                else:
                    token = account_info["token"]
        else:
            raise errors.Invalid("인자값이 더 많이왔어요.")

        guild_info_response = self.get_emoji_list(token, int(guild_id))  # 토큰과 함께 get요청함

        if guild_info_response.status_code != 200:
            raise errors.Invalid("정상적인 토큰이 아니거나 서버를 찾을수없어요. 맞는 토큰인지, 해당 서버에 접속해있는지 확인해주세요.")
        else:
            guild_info = guild_info_response.json()

        if guild_info["emojis"]:
            base_url = "https://cdn.discordapp.com/emojis/"
            for emoji in guild_info["emojis"]:  # 이모지 리스트로 가져옴
                if emoji["animated"] is True:  # 만약 gif면 gif 다운로드
                    param = emoji["id"] + ".gif"
                else:  # 아닐경우 png로
                    param = emoji["id"] + ".png"

                self.title = clean_title(
                    f'{guild_info["name"]}({guild_info["id"]})'  # 폴더 이름은 서버 이름, id
                )
                self.urls.append(base_url + param + "?v=1")  # 인자 합치기
        else:
            raise errors.Invalid("해당 서버에는 이모지가 없어요")

    def get_emoji_list(self, token: str, guild_id: int) -> dict:
        response = requests.get(
            f"https://discordapp.com/api/v6/guilds/{guild_id}",
            headers={"Authorization": token},
        )
        if response.status_code == 401:
            response = requests.get(
                f"https://discordapp.com/api/v6/guilds/{guild_id}",
                headers={"Authorization": f"Bot {token}"},
            )

        return response

    def post_account_info(self, email: str, password: str) -> dict:
        response = requests.post(
            "https://discordapp.com/api/v8/auth/login",
            json={
                "email": email,
                "password": password,
                "undelete": False,
                "captcha_key": None,
                "login_source": None,
                "gift_code_sku_id": None,
            },
        )

        return response



File: src/extractor/xhamster_downloader.py

import downloader, ree as re
from utils import Downloader, Session, LazyUrl, get_print, get_ext, try_n, format_filename, clean_title, get_resolution
from translator import tr_
from io import BytesIO
import ytdl



class Downloader_xhamster(Downloader):
    type = 'xhamster'
    __name = r'([^/]*\.)?(xhamster|xhwebsite|xhofficial|xhlocal|xhopen|xhtotal|megaxh|xhwide|xhtab|xhtime)([0-9]*)' #3881, #4332, #4826, #5029, #5696, #5893
    URLS = [rf'regex:{__name}\.[a-z0-9]+/(videos|users|creators|photos/gallery)/']
    single = True
    display_name = 'xHamster'
    ACCEPT_COOKIES = __name

    def init(self):
        if re.search(r'xhamsterlive[0-9]*\.', self.url):
            raise Exception('xHamsterLive')
        if not re.search(r'{}\.'.format(self.__name), self.url):
            self.url = 'https://xhamster.com/videos/{}'.format(self.url)
        self.session = Session('chrome')

    @classmethod
    def fix_url(cls, url):
        url = re.sub(r'(/users/[^/]+/videos)/[0-9]+', r'\1', url, 1) #5029
        return url

    @classmethod
    def key_id(cls, url):
        return re.sub(cls.__name+r'\.[^/]+', 'domain', url, 1).replace('http://', 'https://')

    def read(self):
        cw = self.cw
        self.enableSegment(1024*1024//2)
        thumb = BytesIO()

        if '/users/' in self.url or '/creators/' in self.url: #6257
            info = read_channel(self.url, self.session, cw)
            urls = info['urls']
            videos = [Video(url) for url in urls]
            video = self.process_playlist(info['title'], videos)
        elif '/photos/gallery/' in self.url:
            info = read_gallery(self.url, self.session, cw)
            for img in info['imgs']:
                self.urls.append(img.url)
            self.single = False
            self.title = clean_title(info['title'])
            self.url = info['url']
            self.disableSegment()
            return
        else:
            video = Video(self.url)
            video.url()
            self.urls.append(video.url)
            self.title = video.title

        downloader.download(video.info['thumbnail'], buffer=thumb)
        self.setIcon(thumb)


class Video:
    _url = None

    def __init__(self, url):
        #url = downloader.real_url(url)
        self.url = LazyUrl(url, self.get, self)

    @try_n(2)
    def get(self, url):
        if self._url is None:
            self.info = get_info(url)

            self.title = self.info['title']
            id = self.info['id']

            #4773
            fs = self.info['formats']
            res = max(get_resolution(), min(f['height'] for f in fs))
            fs = [f for f in fs if f['height'] <= res]

            video_best = fs[-1]
            self._url = video_best['url']
            ext = get_ext(self._url)
            self.filename = format_filename(self.title, id, ext)

            if isinstance(self._url, str) and 'referer=force' in self._url.lower():
                self._referer = self._url
            else:
                self._referer = url
        return self._url, self._referer


def get_info(url): #6318
    info = {}
    ydl = ytdl.YoutubeDL()
    d = ydl.extract_info(url)
    info['title'] = d['title']
    info['id'] = d['id']
    info['thumbnail'] = d['thumbnail']
    fs = []
    for f in d['formats']:
        if f['protocol'] != 'https':
            continue
        f = {'url': f['url'], 'height': f['height']}
        fs.append(f)
    fs = sorted(fs, key=lambda f: f['height'])
    info['formats'] = fs
    return info


def read_page(type_, username, p, session, cw):
    print_ = get_print(cw)
    if type_ == 'users':
        url = f'https://xhamster.com/users/{username}/videos/{p}'
    elif type_ == 'creators':
        url = f'https://xhamster.com/creators/{username}/exclusive/{p}'
    else:
        raise NotImplementedError(type_)
    print_(url)
    n = 4
    for try_ in range(n):
        try:
            soup = downloader.read_soup(url, session=session)
            items = soup.findAll('div', class_='thumb-list__item')
            if not items and try_ < n-1:
                continue
            break
        except Exception as e:
            e_ = e
            print(e)
    else:
        if p == 1:
            raise e_
        else:
            return []
    return items


def read_channel(url, session, cw=None):
    type_, username = re.find(r'/(users|creators)/([^/]+)', url, err='no username')

    info = {}
    soup = downloader.read_soup(url, session=session)
    title = (soup.find('div', class_='user-name') or soup.find('h1')).text.strip()
    info['title'] = '[Channel] {}'.format(title)

    urls = []
    urls_set = set()
    for p in range(1, 101):
        items = read_page(type_, username, p, session, cw)
        if not items:
            print('no items')
            break
        for item in items:
            if item.find('span', class_='thumb-image-container__status-text'): #2858
                continue
            url = item.a.attrs['href']
            if url in urls_set:
                print('duplicate:', url)
                continue
            urls_set.add(url)
            urls.append(url)
        s = '{} {} - {}'.format(tr_('읽는 중...'), info['title'], len(urls))
        if cw:
            cw.setTitle(s)
        else:
            print(s)

    info['urls'] = urls

    return info


class Image:
    def __init__(self, url, id, referer):
        self.id = id
        self._url = url
        self.url = LazyUrl(referer, self.get, self)

    def get(self, referer):
        url = self._url
        ext = get_ext(url)
        self.filename = '{}{}'.format(self.id, ext)
        return url


def setPage(url, p):
    url = url.strip('/')
    c = url.split('/photos/gallery/')[1].count('/')
    if c:
        url = '/'.join(url.split('/')[:-1])
    if p > 1:
        url += '/{}'.format(p)
    return url


def read_gallery(url, session, cw=None):
    print_ = get_print(cw)

    info = {}

    soup = downloader.read_soup(url, session=session)

    h1 = soup.find('h1')
    if h1.find('a'):
        url = h1.find('a')['href']
        return read_gallery(url, session, cw)
    info['title'] = h1.text.strip()
    info['url'] = setPage(url, 1)

    imgs = []
    ids = set()
    for p in range(1, 101):
        print_('p: {}'.format(p))
        url = setPage(url, p)
        soup = downloader.read_soup(url, session=session)

        view = soup.find('div', id='photo-slider')
        photos = view.findAll('a', id=lambda id: id and id.startswith('photo-'))

        if not photos:
            print_('no photos')
            break

        for photo in photos:
            img = photo['href']
            id = photo['id'].split('photo-')[1]
            referer = url
            if id in ids:
                print('duplicate:', id)
                continue
            ids.add(id)
            img = Image(img, id, referer)
            imgs.append(img)

    info['imgs'] = imgs

    return info



File: src/extractor/nhentai_com_downloader.py

#coding:utf8
import downloader
import ree as re
from utils import urljoin, LazyUrl, Downloader, try_n, join, json
import os



class Downloader_nhentai_com(Downloader):
    type = 'nhentai_com'
    URLS = [r'regex:https?://nhentai.com']
    MAX_CORE = 16
    display_name = 'nhentai.com'
    ACCEPT_COOKIES = [r'(.*\.)?nhentai\.com']

    def init(self):
        self.info = get_info(self.url)
        self.url = self.info['url']

    @classmethod
    def key_id(cls, url):
        url = url.lower()
        return re.find(r'/comic/([^/?]+)', url) or url

    def read(self):
        info = self.info

        artist = join(info['artists'])
        self.artist = artist if info['artists'] else None
        group = join(info['groups'])
        lang = info['lang'] or 'N／A'
        series = info['seriess'][0] if info['seriess'] else 'N／A'
        title = self.format_title(info['type'], info['id'], info['title'], artist, group, series, lang)

        for img in info['imgs']:
            self.urls.append(img.url)

        self.title = title


@LazyUrl.register
class LazyUrl_nhentai_com(LazyUrl):
    type = 'nhentai_com'
    def dump(self):
        referer = self._url
        url = self.image.url_img
        return {
            'referer': referer,
            'url': url,
            'p': self.image.p,
            }
    @classmethod
    def load(cls, data):
        referer = data['referer']
        url = data['url']
        img = Image(referer, url, data['p'])
        return img.url


class Image:
    def __init__(self, url_page, url_img, p):
        self.p = p
        self.referer = url_page
        self.filename = os.path.basename(url_img)
        self.url_img = url_img
        self.url = LazyUrl_nhentai_com(url_page, lambda _: self.url_img, self)


@try_n(4)
def get_info(url):
    url = downloader.real_url(url)
    q = re.find(r'/comic/([^/?]+)', url)

    url_api = 'https://nhentai.com/api/comics/{}'.format(q)
    data_raw = downloader.read_html(url_api, url)
    data = json.loads(data_raw)

    url_api = 'https://nhentai.com/api/comics/{}/images'.format(q)
    data_raw = downloader.read_html(url_api, url)
    data_images = json.loads(data_raw)

    info = {}
    info['url'] = url

    info['id'] = int(data['id'])
    info['type'] = data['category']['name']
    info['title'] = data['title']
    info['artists'] = [x['name'] for x in data['artists']]
    info['groups'] = [x['name'] for x in data['groups']]
    info['seriess'] = [x['name'] for x in data['parodies']]
    info['lang'] = data['language']['name']

    imgs = []
    for img in data_images['images']:
        img = urljoin(url, img['source_url'])
        img = Image(url, img, len(imgs))
        imgs.append(img)
    info['imgs'] = imgs

    return info



File: src/extractor/xvideo_downloader.py

import downloader
from utils import Downloader, Soup, LazyUrl, urljoin, format_filename, Session, get_ext, get_print, get_max_range, html_unescape, try_n, limits, json
from io import BytesIO
import ree as re
from m3u8_tools import playlist2stream
from translator import tr_
from timee import sleep
CHANNEL_PATTERN = r'/(profiles|[^/]*channels)/([0-9a-zA-Z_-]+)'


def get_id(url):
    url = url.lower()
    if '/prof-video-click/' in url:
        return url.split('/prof-video-click/')[1].split('/')[2]
    return re.find(r'xvideos[0-9]*\.[^/]+/video([0-9]+)', url, err='no id')


class Video:
    _url = None

    def __init__(self, url_page):
        url_page = Downloader_xvideo.fix_url(url_page)
        self.url = LazyUrl(url_page, self.get, self)

    def get(self, url_page):
        if not self._url:
            self._get(url_page)
        return self._url

    @try_n(4)
    @limits(2)
    def _get(self, url_page):
        id = get_id(url_page)
        html = downloader.read_html(url_page)
        soup = Soup(html)
        self.title = html_unescape(soup.find('title').text).replace('- XVIDEOS.COM', '').strip()
        url = re.find(r'''.setVideoHLS\(['"](.+?)['"]\)''', html) or re.find(r'''.setVideoUrlHigh\(['"](.+?)['"]\)''', html) or re.find(r'''.setVideoUrlLow\(['"](.+?)['"]\)''', html) #https://www.xvideos.com/video65390539/party_night
        if not url:
            raise Exception('no video url')
        ext = get_ext(url)
        if ext.lower() == '.m3u8':
            url = playlist2stream(url, n_thread=5) #4773
        self.url_thumb = soup.find('meta', {'property': 'og:image'}).attrs['content']
        self.filename = format_filename(self.title, id, '.mp4')
        self._url= url

    @property
    def thumb(self):
        self.url()
        f = BytesIO()
        downloader.download(self.url_thumb, buffer=f)
        return f



class Downloader_xvideo(Downloader):
    type = 'xvideo'
    URLS = [r'regex:[./]xvideos[0-9]*\.(com|in|es)']
    single = True
    display_name = 'XVideos'
    ACCEPT_COOKIES = [r'(.*\.)?xvideos[0-9]*\.(com|in|es)']

    def init(self):
        if 'xvideos.' in self.url.lower():
            self.url = self.url.replace('http://', 'https://')
        else:
            self.url = 'https://www.xvideos.com/{}'.format(self.url)

    @classmethod
    def fix_url(cls, url):
        url = re.sub(r'[^/]*xvideos[0-9]*\.[^/]+', 'www.xvideos.com', url).replace('http://', 'https://')
        url = url.replace('/THUMBNUM/', '/')
        return url

    @classmethod
    def key_id(cls, url):
        res = re.find(CHANNEL_PATTERN, url)
        if res:
            return '_'.join(res)
        return url

    def read(self):
        res = re.find(CHANNEL_PATTERN, self.url)
        if res:
            header, username = res
            info = read_channel(self.url, self.cw)
            videos = [Video(url) for url in info['urls']]
            video = self.process_playlist('[Channel] {}'.format(info['name']), videos)
        else:
            video = Video(self.url)
            video.url()
            self.title = video.title
            self.urls.append(video.url)

        self.setIcon(video.thumb)


def read_channel(url_page, cw=None):
    print_ = get_print(cw)
    res = re.find(CHANNEL_PATTERN, url_page)
    if res is None:
        raise Exception('Not channel')
    header, username = res
    print(header, username)
    max_pid = get_max_range(cw)
    info = {}
    info['header'] = header
    info['username'] = username
    session = Session()
    urls = []
    ids = set()
    for p in range(100):
        url_api = urljoin(url_page, '/{}/{}/videos/best/{}'.format(header, username, p))
        print_(url_api)
        r = session.post(url_api)
        data = json.loads(r.text)

        videos = data.get('videos') #4530
        if not videos:
            print_('empty')
            break

        for video in videos:
            id_ = video['id']
            if id_ in ids:
                print_('duplicate: {}'.format(id_))
                continue
            ids.add(id_)
            info['name'] = video['pn']
            urls.append(urljoin(url_page, video['u']))

        if len(urls) >= max_pid:
            break

        n = data['nb_videos']

        s = '{} {} - {}'.format(tr_('읽는 중...'), info['name'], len(urls))
        if cw:
            cw.setTitle(s)
        else:
            print(s)
        if len(ids) >= n:
            break
        sleep(1, cw)
    if not urls:
        raise Exception('no videos')
    info['urls'] = urls[:max_pid]
    return info



File: src/extractor/mrm_downloader.py

#coding:utf8
from utils import Soup, urljoin, LazyUrl, Downloader, try_n, get_print, clean_title, get_ext, check_alive
from translator import tr_
import ree as re
import clf2#


class Image:
    def __init__(self, url, p, page, cw):
        self.cw = cw
        ext = get_ext(url)
        self.filename = '{:04}{}'.format(p, ext)
        if page.title is not None:
            self.filename = '{}/{}'.format(page.title, self.filename)
        self._url = url
        self.url = LazyUrl(page.url, self.get, self)

    def get(self, _):
        return self._url#'tmp://' + clf2.download(self._url, cw=self.cw)


class Page:
    def __init__(self, title, url, soup=None):
        self.title = clean_title(title)
        self.url = url
        self.soup = soup



class Downloader_mrm(Downloader):
    type = 'mrm'
    URLS = ['myreadingmanga.info']
    _soup = None
    MAX_CORE = 4
    display_name = 'MyReadingManga'
    ACCEPT_COOKIES = [r'(.*\.)?myreadingmanga\.info']

    def init(self):
        self.session = get_session(self.url, self.cw)

    @classmethod
    def fix_url(cls, url):
        return re.find('https?://myreadingmanga.info/[^/]+', url, err='err')

    @property
    def soup(self):
        if self._soup is None:
            for try_ in range(8):
                try:
                    html = read_html(self.url, session=self.session, cw=self.cw)
                    break
                except Exception as e:
                    e_ = e
                    self.print_(e)
            else:
                raise e_
            self._soup = Soup(html)
        return self._soup

    @property
    def name(self):
        title = get_title(self.soup)
        return title

    def read(self):
        self.title = '읽는 중... {}'.format(self.name)

        imgs = get_imgs(self.url, self.soup, self.session, self.cw)

        for img in imgs:
            self.urls.append(img.url)

        self.title = self.name


def get_title(soup):
    title = soup.find('h1', class_='entry-title').text.strip()
    title = fix_title(title)
    title = clean_title(title)
    return title


def get_imgs(url, soup=None, session=None, cw=None):
    if soup is None:
        html = read_html(url, session=session, cw=cw)
        soup = Soup(html)

    title = get_title(soup)

    pagination = soup.find('div', class_='pagination')

    if pagination is None:
        page = Page(None, url, soup)
        imgs = get_imgs_page(page, session=session, cw=cw)
    else:
        pages = get_pages(url, soup, session=session)
        imgs = []
        for i, page in enumerate(pages):
            check_alive(cw)
            s = '{} {} / {}  ({} / {})'.format(tr_('읽는 중...'), title, page.title, i+1, len(pages))
            if cw:
                cw.setTitle(s)
            else:
                print(s)

            imgs += get_imgs_page(page, session=session, cw=cw)

    if not imgs:
        raise Exception('no imgs')

    return imgs


def get_pages(url, soup=None, session=None):
    if soup is None:
        html = read_html(url, session=session, cw=None)
        soup = Soup(html)
    pagination = soup.find('div', class_='pagination')

    pages = []
    hrefs = set()
    for a in pagination.findAll('a'):
        href = a.attrs.get('href', '')
        href = urljoin(url, href)
        if not href.startswith(url):
            print('not match', href)
            continue
        while href.endswith('/'):
            href = href[:-1]
        if href in hrefs:
            print('duplicate', href)
            continue
        hrefs.add(href)
        text = a.text.strip()
        page = Page(text, href)
        pages.append(page)

    if url not in hrefs:
        page = Page('1', url, soup)
        pages.insert(0, page)

    return pages


@try_n(4)
def get_imgs_page(page, session=None, cw=None):
    url = page.url
    soup = page.soup
    if soup is None:
        html = read_html(url, session=session, cw=None)
        soup = Soup(html)
        page.soup = soup

    view = soup.find('div', class_='entry-content')

    imgs = []
    for img in view.findAll('img'):
        img = img.attrs.get('data-lazy-src') or img.attrs.get('data-src')
        if img is None:
            continue
        img = urljoin(url, img)
        img = Image(img, len(imgs), page, cw)
        imgs.append(img)
    print(page.title, len(imgs), page.url)

    return imgs


def fix_title(title):
    title = re.sub(r'\(?[^()]*?c\.[^() ]+\)?', '', title)
    while '  ' in title:
        title = title.replace('  ', ' ')
    return title


def read_html(url, session, cw):
##    html = downloader.read_html(url, session=session)
##    soup = Soup(html)
##
##    cf = soup.find('div', class_='cf-browser-verification')
##    if cf is None:
##        return html

    r = clf2.solve(url, cw=cw, session=session)

    return r['html']


@try_n(4)
def get_session(url, cw=None):
    print_ = get_print(cw)
##    html = downloader.read_html(url)
##    soup = Soup(html)
##
##    cf = soup.find('div', class_='cf-browser-verification')
##    if cf is None:
##        print_('no cf protection')
##        return None

    print_('cf protection')
    r = clf2.solve(url, cw=cw)
    session = r['session']

    return session



File: src/extractor/iwara_downloader.py

import downloader
from utils import Soup, urljoin, Downloader, LazyUrl, get_print, clean_url, clean_title, check_alive, Session, try_n, format_filename, tr_, get_ext, print_error, get_max_range
import ree as re
import errors
import clf2
import hashlib
import urllib
from io import BytesIO
from timee import time
TIMEOUT = 300
PATTERN_ID = r'(image|video)/([0-9a-zA-Z_-]+)'



class Downloader_iwara(Downloader):
    type = 'iwara'
    URLS = ['iwara.tv']
    MAX_CORE = 16#
    single = True
    display_name = 'Iwara'
    ACCEPT_COOKIES = [r'(.*\.)?iwara\.tv']

    @classmethod
    def fix_url(cls, url):
        url = clean_url(url)
        return url.split('?')[0]

    def init(self):
        self.session = Session()
        self.setTimeout(TIMEOUT)

    def read(self):
        info = get_info(self.url, self.session, self.cw)
        if info is None:
            return # embeded
        self.title = clean_title(info['title'])

        videos = info['files']
        self.single = len(videos) < 2

        # first video must be valid
        while videos:
            video = videos[0]
            try:
                video.url()
                break
            except Exception as e:
                e_ = e
                self.print_(print_error(e))
                videos.remove(video)
        else:
            raise e_

        if info.get('playlist', False):
            video = self.process_playlist(info['title'], videos)
        else: #6031
            self.urls += [file.url for file in videos]

        self.enableSegment()

        url_thumb = video.url_thumb
        self.print_(f'url_thumb: {url_thumb}')
        if url_thumb:
            f = BytesIO()
            downloader.download(url_thumb, buffer=f, session=self.session, customWidget=self.cw)
            f.seek(0)
            self.setIcon(f)

        username = info.get('username')
        if username:
            self.artist = username



class File:
    def __init__(self, type, url, referer, info, session, multi_post=False):
        title = info['title']
        p = len(info['files'])
        self.url = LazyUrl(referer, lambda _: url, self)
        ext = get_ext(url) or downloader.get_ext(url, session=session)
        if type == 'video':
            id_ = re.find(PATTERN_ID, referer, err='no video id')[1]
            self.filename = format_filename(title, id_, ext) #4287
        else:
            name = '{}_p{}'.format(clean_title(title), p) if multi_post else p
            self.filename = '{}{}'.format(name, ext)
        self.url_thumb = info.get('url_thumb')


class LazyFile:
    def __init__(self, url, session, cw):
        self.session = session
        self.cw = cw
        self.url = LazyUrl(url, self.get, self)

    def get(self, url):
        info = get_info(url, self.session, self.cw)
        file = info['files'][0]
        self.filename = file.filename
        self.url_thumb = file.url_thumb
        return file.url()


def get_token(session, cw=None):
    token = None
    def f(html, browser=None):
        def callback(r):
            nonlocal token
            token = r
        browser.runJavaScript('window.localStorage.getItem("token")', callback=callback)
        return bool(token)
    clf2.solve('https://iwara.tv', session=session, cw=cw, f=f, timeout=15)
    #print_(f'token: {token}')
    r = session.post('https://api.iwara.tv/user/token', headers={'Authorization': f'Bearer {token}'})
    d = r.json()
    token = d['accessToken']
    #print_(f'token2: {token}')
    return token


@try_n(2)
def get_info(url, session, cw, multi_post=False):
    print_ = get_print(cw)
    t0 = None
    def f(html, browser=None):
        nonlocal t0
        soup = Soup(html)
        if t0 is None:
            t0 = time()
        if time() - t0 > 10 or '/profile/' in url.lower():
            for a in soup.findAll('a'):
                if urljoin(url, a.get('href', '')) == urljoin(url, '/login'):
                    raise errors.LoginRequired(method='browser', url='https://www.iwara.tv/login', cookie=False, w=1460) #5794
        buttons = soup.findAll(class_='button--primary')
        if buttons:
            for i, button in enumerate(buttons):
                button_text = button.text
                if not button_text:
                    continue
                print_(f'button: {button_text}')
                if button_text.lower() in ['i am over 18', 'continue']:
                    browser.runJavaScript(f'btns=document.getElementsByClassName("button--primary");btns[{i}].click();') #5794#issuecomment-1517879513
        if '/profile/' in url.lower():
            return soup.find('div', class_='page-profile__header') is not None
        else:
            details = soup.find('div', class_='page-video__details')
            if details and not soup.find('div', class_='vjs-poster') and not soup.find(class_='embedPlayer__youtube'): #6737, #6836
                print_('no poster')
                return False
            details = details or soup.find('div', class_='page-image__details')
            return details is not None and details.find('div', class_='text--h1') is not None

    html = clf2.solve(url, session=session, f=f, cw=cw, timeout=30)['html'] #5794
    soup = Soup(html)

    info = {}
    info['files'] = []

    type = url.split('/')[3]
    if type == 'profile':
        max_pid = get_max_range(cw)
        ids = set()
        sub = (url+'/').split('/')[5]
        if not sub:
            sub = 'videos'
        uid = url.split('/')[4]
        url_api = f'https://api.iwara.tv/profile/{uid}'
        j = downloader.read_json(url_api, session=session)
        info['username'] = username = j['user']['name']
        info['id'] = id = j['user']['username']
        info['title'] = f'[Channel] [{sub.capitalize()}] {username} ({id})'
        id = j['user']['id']
        if sub == 'videos':
            info['playlist'] = True
            for p in range(100):
                url_api = f'https://api.iwara.tv/videos?page={p}&sort=date&user={id}'
                j = downloader.read_json(url_api, session=session)
                for post in j['results']:
                    id_ = post['id']
                    if id_ in ids:
                        continue
                    ids.add(id_)
                    slug = post['slug']
                    url_post = f'https://www.iwara.tv/video/{id_}/{slug}'
                    file = LazyFile(url_post, session, cw)
                    info['files'].append(file)
                if cw: cw.setTitle(tr_('읽는 중... {} ({} / {})').format(info['title'], len(ids), j['count']))
                if len(info['files']) >= max_pid:
                    break
                if j['limit']*(p+1) >= j['count']: break
        elif sub == 'images':
            for p in range(100):
                url_api = f'https://api.iwara.tv/images?page={p}&sort=date&user={id}'
                j = downloader.read_json(url_api, session=session)
                for post in j['results']:
                    check_alive(cw)
                    id_ = post['id']
                    if id_ in ids:
                        continue
                    ids.add(id_)
                    slug = post['slug']
                    url_post = f'https://www.iwara.tv/image/{id_}/{slug}'
                    info_post = get_info(url_post, session, cw, True)
                    info['files'] += info_post['files']
                    print_(f'imgs: {len(info["files"])}')
                    if cw: cw.setTitle(tr_('읽는 중... {} ({} / {})').format(info['title'], len(ids), j['count']))
                    if len(info['files']) >= max_pid:
                        break
                if len(info['files']) >= max_pid:
                        break
                if j['limit']*(p+1) >= j['count']: break
        else:
            raise NotImplementedError(f'profile: {sub}')
        return info

    details = soup.find('div', class_='page-video__details') or soup.find('div', class_='page-image__details')
    info['title'] = details.find('div', class_='text--h1').text.strip()
    info['username'] = soup.find('a', class_='username')['title']

    soup.find('div', class_='videoPlayer') or soup.find('div', class_='page-image__slideshow')

    id = re.find(PATTERN_ID, url, err='no id')[1]

    try:
        token = get_token(session, cw=cw)
    except Exception as e:
        print_(print_error(e))
        token = None

    url_api = f'https://api.iwara.tv/{type}/{id}'
    hdr = {}
    if token:
        hdr['authorization'] = f'Bearer {token}'
    data = downloader.read_json(url_api, url, session=session, headers=hdr)

    if data.get('embedUrl'):
        if cw and not cw.downloader.single:
            raise errors.Invalid('[iwara] Embeded: {}'.format(data['embedUrl']))
        #5869
        cw.downloader.pass_()
        cw.gal_num = cw.url = data['embedUrl']
        d = Downloader.get('youtube')(data['embedUrl'], cw, cw.downloader.thread, 1)
        d.start()
        return

    if not data.get('files'):
        data['files'] = [data['file']]

    for file in data['files']:
        id_ = file['id']
        if type == 'video':
            fileurl = data['fileUrl']
            up = urllib.parse.urlparse(fileurl)
            q = urllib.parse.parse_qs(up.query)
            paths = up.path.rstrip('/').split('/')
            x_version = hashlib.sha1('_'.join((paths[-1], q['expires'][0], '5nFp9kmbNnHdAFhaqMvt')).encode()).hexdigest() # https://github.com/yt-dlp/yt-dlp/issues/6549#issuecomment-1473771047
            j = downloader.read_json(fileurl, url, session=session, headers={'X-Version': x_version})
            def key(x):
                if x['name'].lower() == 'source':
                    return float('inf')
                try:
                    return float(x['name'])
                except:
                    return -1
            x = sorted(j, key=key)[-1]
            print_(f'name: {x["name"]}')
            url_file = urljoin(url, x['src']['view'])
            poster = soup.find('div', class_='vjs-poster')['style']
            info['url_thumb'] = urljoin(url, re.find(r'url\("(.+?)"', poster, err='no poster'))
        else:
            name = file['name']
            url_file = f'https://i.iwara.tv/image/original/{id_}/{name}'
        if len(data['files']) == 1:
            multi_post = True#
        file = File(type, url_file, url, info, session, multi_post)
        info['files'].append(file)

    return info



File: src/extractor/soundcloud_downloader.py

#coding: utf8
import downloader
from io import BytesIO
from utils import Downloader, LazyUrl, get_print, try_n, lock, get_max_range, format_filename, limits
import ffmpeg
import ytdl
from m3u8_tools import M3u8_stream
CLIENT_ID = None


@lock
def get_cid(force=False):
    global CLIENT_ID
    if CLIENT_ID is None or force:
        print('update cid...')
        d = ytdl.YoutubeDL()
        e = ytdl.extractor.soundcloud.SoundcloudIE(d)
        e._update_client_id()
        CLIENT_ID = e._CLIENT_ID
    return CLIENT_ID


class Audio:
    _url = None

    def __init__(self, url, album_art, cw=None):
        self.album_art = album_art
        self.cw = cw
        self.url = LazyUrl(url, self.get, self, pp=self.pp)

    @try_n(2)
    @limits(1)
    def get(self, url):
        print_ = get_print(self.cw)
        if self._url:
            return self._url

        ydl = ytdl.YoutubeDL()
        self.info = info = ydl.extract_info(url)

        formats = info['formats']
        print(formats)
        def key(f):
            abr = f.get('abr')
            if abr is None:
                abr = 320
            return int(abr)
        formats = sorted(formats, key=key, reverse=True)
        url_audio = None

        for format in formats:
            protocol = format['protocol']
            print_('【{}】 format【{}】 abr【{}】'.format(protocol, format['format'], format.get('abr', 0)))
            if not url_audio and protocol in ['http', 'https']:
                url_audio = format['url']

        if not url_audio:
            url_audio = M3u8_stream(formats[0]['url'])
            self.album_art = False#

        self.username = info['uploader']
        self.title = '{} - {}'.format(self.username, info['title'])
        self.filename = format_filename(self.title, '', '.mp3')

        self._thumb = None
        def thumb():
            if self._thumb is None:
                for t in info['thumbnails'][::-1]:
                    width = t.get('width', 1080)
                    if not 100 <= width <= 500:
                        continue
                    url_thumb = t['url']
                    f = BytesIO()
                    try:
                        downloader.download(url_thumb, buffer=f)
                        break
                    except Exception as e:
                        print(e)
                        f = None
                self._thumb = f
            else:
                f = self._thumb
            if f is not None:
                f.seek(0)
            return f
        self.thumb = thumb

        self._url = url_audio
        return self._url

    def pp(self, filename):
        if self.thumb() and self.album_art:
            ffmpeg.add_cover(filename, self.thumb(), {'artist':self.username, 'title':self.info['title']}, cw=self.cw)



class Downloader_soundcloud(Downloader):
    type = 'soundcloud'
    single = True
    URLS = ['soundcloud.com']
    #lock = True
    audio = None
    display_name = 'SoundCloud'
    ACCEPT_COOKIES = [r'(.*\.)?soundcloud\.com']

    def init(self):
        if 'soundcloud.com' in self.url.lower():
            self.url = self.url.replace('http://', 'https://')
        else:
            self.url = 'https://soundcloud.com/{}'.format(self.url)

    @classmethod
    def fix_url(cls, url):
        return url.split('?')[0]

    def read(self):
        album_art = self.ui_setting.albumArt.isChecked()
        info = get_audios(self.url, self.cw, album_art)
        audios = info['audios']

        if not audios:
            raise Exception('no audios')

        # first audio must be valid
        while audios:
            audio = audios[0]
            try:
                audio.url()
                break
            except Exception as e:
                e_ = e
                print(e)
                audios.remove(audio)
        else:
            raise e_

        if len(audios) > 1:
            audio = self.process_playlist(info['title'], audios)
        else:
            self.urls.append(audio.url)
            self.title = audio.title

        self.artist = audio.username
        self.setIcon(audio.thumb())


@try_n(2)
def get_audios(url, cw, album_art):
    print_ = get_print(cw)
    url = url.rstrip('/')
    if url.count('/') == 3:
        url += '/tracks'

    options = {
        'extract_flat': True,
        'playlistend': get_max_range(cw),
        }

    ydl = ytdl.YoutubeDL(options, cw=cw)
    info = ydl.extract_info(url)
    if 'entries' in info:
        entries = info['entries']
        title = info['title']
        for _type in ['All', 'Tracks', 'Albums', 'Sets', 'Reposts', 'Likes', 'Spotlight']:
            x = '({})'.format(_type)
            if x in title:
                title = title.replace(x, '')
                kind = _type
                break
        else:
            kind = 'Playlist'
        print_('kind: {}'.format(kind))
        info['title'] = '[{}] {}'.format(kind.capitalize(), title)
    else:
        entries = [info]

    audios = []
    for e in entries:
        url = e.get('webpage_url') or e['url']
        if '/sets/' in url:
            continue
        audio = Audio(url, album_art, cw=cw)
        audios.append(audio)

    info['audios'] = audios

    return info



File: src/extractor/vimeo_downloader.py

import downloader
from io import BytesIO as IO
from utils import Downloader, LazyUrl, get_ext, format_filename, try_n
import ytdl



class Downloader_vimeo(Downloader):
    type = 'vimeo'
    URLS = ['vimeo.com']
    single = True
    ACCEPT_COOKIES = [r'(.*\.)?vimeo\.com']

    def init(self):
        if 'vimeo.com' not in self.url.lower():
            self.url = 'https://vimeo.com/{}'.format(self.url)

    def read(self):
        video = Video(self.url, cw=self.cw)
        video.url()#

        self.urls.append(video.url)
        self.setIcon(video.thumb)

        self.enableSegment()

        self.title = video.title


class Video:
    _url = None

    def __init__(self, url, cw=None):
        self.url = LazyUrl(url, self.get, self)
        self.cw = cw

    @try_n(4)
    def get(self,  url):
        if self._url:
            return self._url

        ydl = ytdl.YoutubeDL(cw=self.cw)
        info = ydl.extract_info(url)
        fs = [f for f in info['formats'] if f['protocol'] in ['http', 'https']]
        fs = sorted(fs, key=lambda f: int(f.get('width', 0)), reverse=True)
        if not fs:
            raise Exception('No MP4 videos')
        f = fs[0]

        self.thumb_url = info['thumbnails'][0]['url']
        self.thumb = IO()
        downloader.download(self.thumb_url, buffer=self.thumb)
        self.title = info['title']
        url_video = f['url']
        ext = get_ext(url) or '.mp4'
        self.filename = format_filename(self.title, info['id'], ext)
        self._url = url_video
        return self._url



File: src/extractor/sankaku_downloader.py

#coding: utf-8
#https://chan.sankakucomplex.com/
#https://idol.sankakucomplex.com/
#https://beta.sankakucomplex.com/
#https://sankaku.app/
#http://white.sankakucomplex.com/
#https://www.sankakucomplex.com/
import downloader
import ree as re
from utils import Downloader, urljoin, query_url, get_max_range, get_print, Soup, lazy, Session, clean_title, check_alive, File, get_ext, limits, clean_url
from translator import tr_
import os
from timee import sleep
from error_printer import print_error
from urllib.parse import quote
import errors
import utils


class File_sankaku(File):
    type = 'sankaku'
    format = 'id'

    def get(self):
        print_ = get_print(self.cw)
        referer = self['referer']

        for try_ in range(4):
            wait(self.cw)
            html = ''
            try:
                html = downloader.read_html(referer, session=self.session)
                soup = Soup(html)
                highres = soup.find(id='highres')
                url = urljoin(referer, highres['href'] if highres else soup.find(id='image')['src'])
                break
            except Exception as e:
                e_msg = print_error(e)
                if '429 Too many requests'.lower() in html.lower():
                    t_sleep = 120 * min(try_ + 1, 2)
                    e = '429 Too many requests... wait {} secs'.format(t_sleep)
                elif 'post-content-notification' in html: # sankaku plus
                    print_('Sankaku plus: {}'.format(self['id']))
                    return ''
                else:
                    t_sleep = 5
                s = '[Sankaku] failed to read image (id:{}): {}'.format(self['id'], e)
                print_(s)
                sleep(t_sleep, self.cw)
        else:
            raise Exception('can not find image (id:{})\n{}'.format(self['id'], e_msg))
        soup = Soup('<p>{}</p>'.format(url))
        url = soup.string
        d = {
            'id': self['id'],
            }
        return {'url': url, 'name': utils.format('sankaku', d, get_ext(url))}



class Downloader_sankaku(Downloader):
    type = 'sankaku'
    URLS = ['chan.sankakucomplex.com', 'idol.sankakucomplex.com', 'www.sankakucomplex.com']
    MAX_CORE = 4
    display_name = 'Sankaku Complex'
    ACCEPT_COOKIES = [r'(.*\.)?(sankakucomplex\.com|sankaku\.app)']

    def init(self):
        type = self.url.split('sankakucomplex.com')[0].split('//')[-1].strip('.').split('.')[-1]
        if type == '':
            type = 'www'
        if type not in ['chan', 'idol', 'www']:
            raise Exception('Not supported subdomain')
        self.type_sankaku = type
        self.url = self.url.replace('&commit=Search', '')
        self.url = clean_url(self.url)
        self.session = Session()

    @lazy
    def soup(self):
        html = downloader.read_html(self.url, session=self.session)
        return Soup(html)

    @classmethod
    def fix_url(cls, url):
        if 'sankakucomplex.com' not in url:
            url = url.replace(' ', '+')
            while '++' in url:
                url = url.replace('++', '+')
            url = quote(url)
            url = url.replace('%2B', '+')
            url = url.replace('%20', '+')#
            if url.startswith('[chan]'):
                type = 'chan'
                url = url.replace('[chan]', '', 1).strip()
            elif url.startswith('[idol]'):
                type = 'idol'
                url = url.replace('[idol]', '', 1).strip()
            elif url.startswith('[www]'):
                type = 'www'
                url = url.replace('[www]', '', 1).strip()
            else:
                raise Exception('Not supported subdomain')
            url = 'https://{}.sankakucomplex.com/?tags={}'.format(type, url)
        return url.replace('http://', 'https://')

    @lazy
    def id(self):
        if self.type_sankaku == 'www':
            id = '[www] ' + self.soup.find('h1', class_='entry-title').text.strip()
        else:
            if '/post/show/' in self.url or '/posts/' in self.url: #6718
                id = get_id(self.url, self.soup)
            else:
                qs = query_url(self.url)
                tags = qs.get('tags', [])
                tags.sort()
                id = ' '.join(tags)
                if not id:
                    id = 'N/A'
            id = '[{}] {}'.format(self.type_sankaku, id)
        return clean_title(id)

    @property
    def name(self):
        return self.id

    def read(self):
        ui_setting = self.ui_setting
        self.title = self.name

        types = ['img', 'gif', 'video']
        if ui_setting.exFile.isChecked():
            if ui_setting.exFileImg.isChecked():
                types.remove('img')
            if ui_setting.exFileGif.isChecked():
                types.remove('gif')
            if ui_setting.exFileVideo.isChecked():
                types.remove('video')

        if self.type_sankaku == 'www':
            imgs = get_imgs_www(self.url, self.soup)
        else:
            info = get_imgs(self.url, self.name, cw=self.cw, types=types, session=self.session)
            self.single = info['single']
            imgs = info['imgs']

        self.urls += imgs

        self.title = self.name


def get_imgs_www(url, soup):
    imgs = []
    view = soup.find('div', class_='entry-content')
    for img in view.findAll('img'):
        img = img.get('data-lazy-src')
        if not img: # no script
            continue
        img = urljoin(url, img)
        if img in imgs:
            print('duplicate', img)
            continue
        imgs.append(img)
    return imgs


def setPage(url, page):
    # Always use HTTPS
    url = url.replace('http://', 'https://')

    # Change the page
    if 'page=' in url:
        url = re.sub(r'page=[0-9]*', 'page={}'.format(page), url)
    else:
        url += '&page={}'.format(page)

    return url


@limits(6)
def wait(cw):
    check_alive(cw)


def get_imgs(url, title=None, cw=None, types=['img', 'gif', 'video'], session=None):
    print_ = get_print(cw)
    print_('types: {}'.format(', '.join(types)))
    if 'chan.sankakucomplex' in url:
        type = 'chan'
    elif 'idol.sankakucomplex' in url:
        type = 'idol'
    else:
        raise Exception('Not supported subdomain')

    info = {}
    info['single'] = False

    if '/post/show/' in url or '/posts/' in url: #6718
        info['single'] = True
        id = get_id(url)
        info['imgs'] = [File_sankaku({'type': type, 'id': id, 'referer': url})]
        return info

    # Range
    max_pid = get_max_range(cw)

    local_ids = {}
    if cw is not None:
        dir = cw.downloader.dir
        try:
            names = os.listdir(dir)
        except Exception as e:
            print(e)
            names = []
        for name in names:
            id = os.path.splitext(name)[0]
            local_ids[id] = os.path.join(dir, name)

    imgs = []
    page = 1
    ids = set()
    url_old = 'https://{}.sankakucomplex.com'.format(type)
    if cw is not None:
        cw.setTitle('{}  {}'.format(tr_('읽는 중...'), title))
    while len(imgs) < max_pid:
        #if page > 25: # Anonymous users can only view 25 pages of results
        #    break
        wait(cw)
        #url = setPage(url, page)
        print_(url)
        try:
            html = downloader.read_html(url, referer=url_old, session=session)
        except Exception as e: #3366
            print_(print_error(e))
            break
        if '429 Too many requests'.lower() in html.lower():
            print_('429 Too many requests... wait 120 secs')
            sleep(120, cw)
            continue
        page += 1
        url_old = url
        soup = Soup(html)
        for banner in soup.findAll('div', class_='has-mail'): #5861
            banner.decompose()
        banner = soup.find('div', class_='popular-previews')
        if banner: #6171
            banner.decompose()
        err = soup.find('div', class_='post-premium-browsing_error')
        if err and not imgs:
            raise errors.LoginRequired(err.text.strip())
        articles = soup.findAll('span', {'class': 'thumb'})

        if not articles:
            if soup.find(class_='post-premium-browsing_error'): #6418
                print_('premium error')
                tags = utils.query_url(url)['tags'][0]
                tags = re.sub(r'id_range:<[0-9]+', '', tags).strip()
                tags += f' id_range:<{min(ids)}'
                url = utils.update_url_query(url, {'tags': tags})
                url = re.sub(r'&page=[0-9]+', '', url)
                url = re.sub(r'&next=[0-9]+', '', url)
                continue
            print_('no articles')
            break

        for article in articles:
            # 1183
            tags = article.find('img', class_='preview')['data-auto_page'].split() #6718
            if 'animated_gif' in tags:
                type_ = 'gif'
            elif 'animated' in tags or 'webm' in tags or 'video' in tags or 'mp4' in tags: # 1697
                type_ = 'video'
            else:
                type_ = 'img'
            if type_ not in types:
                continue

            url_img = article.a['href']
            if not url_img.startswith('http'):
                url_img = urljoin('https://{}.sankakucomplex.com'.format(type), url_img)
            if 'get.sankaku.plus' in url_img: # sankaku plus
                continue
            id = int(re.find(r'p([0-9]+)', article['id'], err='no id')) #5892
            #print_(article)
            if str(id) in local_ids:
                #print('skip', id)
                local = True
            else:
                local = False
            #print(url_img)
            if id not in ids:
                ids.add(id)
                if local:
                    img = local_ids[str(id)]
                else:
                    img = File_sankaku({'type':type, 'id':id, 'referer':url_img})
                imgs.append(img)
                if len(imgs) >= max_pid:
                    break

        try:
            # For page > 50
            pagination = soup.find('div', class_='pagination')
            url = urljoin('https://{}.sankakucomplex.com'.format(type), utils.html.unescape(pagination['next-page-url'])) #6326
##            #3366
##            p = int(re.find(r'[?&]page=([0-9]+)', url, default=1))
##            if p > 100:
##                break
        except Exception as e:
            print_(print_error(e))
            #url = setPage(url, page)
            break

        if cw is not None:
            cw.setTitle('{}  {} - {}'.format(tr_('읽는 중...'), title, len(imgs)))
        else:
            print(len(imgs), 'imgs')

    if not imgs:
        raise Exception('no images')

    info['imgs'] = imgs

    return info


def get_id(url, soup=None):
    if soup is None:
        html = downloader.read_html(url)
        soup = Soup(html)
    if x := soup.find('input', id='post_id'):
        return x['value']
    return soup.find('p', id='hidden_post_id').string



File: src/extractor/asmhentai_downloader.py

#coding: utf8
import downloader
import ree as re
from utils import Soup, urljoin, Downloader, join, Session, File, clean_title, limits
import os
import utils



def get_id(url):
    try:
        return int(url)
    except:
        return int(re.find('/(g|gallery)/([0-9]+)', url)[1])


class File_asmhentai(File):
    type = 'asmhentai'
    format = 'name'

    @limits(.25)
    def get(self):
        soup = downloader.read_soup(self['referer'], self['rereferer'], session=self.session)
        img = soup.find('img', id='fimg')
        url = img['data-src']
        name, ext = os.path.splitext(os.path.basename(url).split('?')[0])
        d = {
            'name': clean_title(name),
            }
        return {'url': url, 'name': utils.format('asmhentai', d, ext)}



class Downloader_asmhentai(Downloader):
    type = 'asmhentai'
    URLS = ['asmhentai.com']
    MAX_CORE = 8
    display_name = 'AsmHentai'
    ACCEPT_COOKIES = [r'(.*\.)?asmhentai\.com']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        id_ = get_id(url)
        return f'https://asmhentai.com/g/{id_}/'

    def read(self):
        info = get_info(self.url, self.session, self.cw)
        self.print_(info)

        # 1225
        artist = join(info['artist'])
        self.artist = artist
        group = join(info['group']) if info['group'] else 'N／A'
        lang = info['language'][0] if info['language'] else 'N／A'
        series = info['parody'][0] if info['parody'] else 'N／A'
        title = self.format_title(info['category'][0], info['id'], info['title'], artist, group, series, lang)

        for i in range(info['n']):
            url = f'https://asmhentai.com/gallery/{info["id"]}/{i+1}/'
            file = File_asmhentai({'referer':url, 'rereferer': self.url})
            self.urls.append(file)

        self.title = title


def get_info(url, session, cw):
    html = downloader.read_html(url, session=session)
    soup = Soup(html)

    info = {}

    info['id'] = get_id(url)

    title = soup.find('h1').text.strip()
    info['title'] = title

    for tag in soup.findAll('span', class_='tag'):
        href = tag.parent.attrs['href']
        href = urljoin(url, href).strip('/')

        key = href.split('/')[3]
        value = href.split('/')[-1]

        if key == 'language' and value == 'translated':
            continue

        if key in info:
            info[key].append(value)
        else:
            info[key] = [value]

    for key in ['artist', 'group', 'parody', 'tag', 'character']:
        if key not in info:
            info[key] = []

    info['n'] = int(soup.find('input', id='t_pages')['value'])

    return info



File: src/extractor/naver_downloader.py

#coding:utf-8
import downloader
import ree as re
from utils import urljoin, Downloader, Soup, LazyUrl, clean_title, get_ext, get_print, Session, json
import errors
PATTERNS = ['.*blog.naver.com/(?P<username>.+)/(?P<pid>[0-9]+)',
            '.*blog.naver.com/.+?blogId=(?P<username>[^&]+).+?logNo=(?P<pid>[0-9]+)',
            '.*?(?P<username>[0-9a-zA-Z_-]+)\.blog\.me/(?P<pid>[0-9]+)']

def get_id(url):
    for pattern in PATTERNS:
        m = re.match(pattern, url)
        if m is None:
            continue
        username = m.group('username')
        pid = m.group('pid')
        break
    else:
        username, pid = None, None
    return username, pid



class Downloader_naver(Downloader):
    type = 'naver'
    URLS = ['blog.naver.', '.blog.me']
    display_name = 'Naver Blog'
    ACCEPT_COOKIES = [r'(.*\.)?naver\.com', r'(.*\.)?blog\.me']

    def init(self):
        self.session = Session()
        username, pid = get_id(self.url)
        if username is None:
            raise errors.Invalid(f'Invalid format: {self.url}')
        self.url = f'https://blog.naver.com/{username}/{pid}'

    @property
    def name(self):
        username, pid = get_id(self.url)
        return clean_title(f'{username}/{pid}')

    def read(self):
        self.title = f'읽는 중... {self.name}'

        imgs = get_imgs(self.url, self.session, self.cw)

        for img in imgs:
            self.urls.append(img.url)

        self.title = self.name


class Image:
    def __init__(self, url, referer, p):
        self.url = LazyUrl(referer, lambda _: url, self)
        #3788, #3817
        ext = get_ext(url)
        self.filename = f'{p:04}{ext}'


class Video:
    def __init__(self, url, referer, p):
        self.url = LazyUrl(referer, lambda _: url, self)
        self.filename = f'video_{p}.mp4'


def read_page(url, session, depth=0):
    print('read_page', url, depth)
    if depth > 10:
        raise Exception('Too deep')
    html = downloader.read_html(url, session=session)

    if len(html) < 5000:
        id = re.find('logNo=([0-9]+)', html, err='no id')
        username = re.find('blog.naver.com/([0-9a-zA-Z]+)', url) or re.find('blogId=([0-9a-zA-Z]+)', url, err='no username')
        url = f'https://m.blog.naver.com/PostView.nhn?blogId={username}&logNo={id}&proxyReferer='

    soup = Soup(html)
    if soup.find('div', {'id': 'viewTypeSelector'}):
        return url, soup
    frame = soup.find('frame')
    if frame is None:
        print('frame is None')
        return read_page(url, session, depth+1)
    return read_page(urljoin('https://blog.naver.com', frame.attrs['src']), session, depth+1)



def get_imgs(url, session, cw):
    print_ = get_print(cw)
    url = url.replace('blog.naver', 'm.blog.naver')
    referer = url
    url_frame, soup = read_page(url, session)

    imgs = []
    urls = set()
    view = soup.find('div', {'id': 'viewTypeSelector'})
    print('view', view is not None)

    imgs_ = view.findAll('span', class_='_img') + view.findAll('img')

    for img in imgs_:
        url = img.attrs.get('src')
        if not url:
            url = img.attrs.get('thumburl')
        if not url:
            continue

        if 'ssl.pstatic.net' in url: #
            continue

        if 'blogpfthumb-phinf.pstatic.net' in url: # profile
            continue

        if 'dthumb-phinf.pstatic.net' in url: # link
            continue

        if 'storep-phinf.pstatic.net' in url: # emoticon
            continue

        url =  url.replace('mblogthumb-phinf', 'blogfiles')
        #url = re.sub('\?type=[a-zA-Z0-9]*', '?type=w1@2x', url)
        #url = re.sub('\?type=[a-zA-Z0-9]*', '', url)
        url = url.split('?')[0]

        if url in urls:
            print('### Duplicate:', url)
            continue

        urls.add(url)
        #url = url.split('?type=')[0]
        img = Image(url, referer, len(imgs))
        imgs.append(img)

    pairs = []

    for video in soup.findAll(class_='_naverVideo'):
        vid = video.attrs['vid']
        key = video.attrs['key']
        pairs.append((vid, key))
    print_(f'pairs: {pairs}')

    for script in soup.findAll('script', class_='__se_module_data'):
        data_raw = script['data-module']
        data = json.loads(data_raw)['data']
        vid = data.get('vid')
        if not vid:
            continue
        key = data['inkey']
        pairs.append((vid, key))

    videos = []
    for vid, key in pairs:
        url_api = f'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{vid}?key={key}'
        data_raw = downloader.read_html(url_api, session=session)
        data = json.loads(data_raw)
        fs = data['videos']['list']
        fs = sorted(fs, key=lambda f: f['size'], reverse=True)
        video = Video(fs[0]['source'], url_frame, len(videos))
        videos.append(video)

    return imgs + videos



File: src/extractor/bili_downloader.py

import downloader
import downloader_v3
from utils import Downloader, get_print, format_filename, clean_title, get_resolution, try_n, Session, uuid, File, get_max_range, query_url
import os
from io import BytesIO
import ffmpeg
import math
import ree as re
import ytdl
import constants
from putils import DIR
import threading
import errors
_VALID_URL = r'''(?x)
                    https?://
                        (?:(?:www|bangumi)\.)?
                        bilibili\.(?:tv|com)/
                        (?:
                            (?:
                                video/[aA][vV]|
                                anime/(?P<anime_id>\d+)/play\#
                            )(?P<id_bv>\d+)|
                            video/[bB][vV](?P<id>[^/?#&]+)
                        )
                    '''


class File_bili(File):
    type = 'bili'
    thread_audio = None

    @try_n(4)
    def get(self):
        session = self.session
        cw = self.cw
        print_ = get_print(cw)

        options = {
                #'noplaylist': True, #5562
                #'extract_flat': True,
                'playlistend': 1,
                }
        ydl = ytdl.YoutubeDL(options, cw=cw)
        info = ydl.extract_info(self['referer'])

        #5562
        entries = info.get('entries')
        if entries:
            info.update(entries[0])

        fs = info['formats']
        res = max(get_resolution(), min(f.get('height', 0) for f in fs))
        print_(f'res: {res}')
        fs = [f for f in fs if f.get('height', 0) <= res]
        for f in fs:
            print_(f"{f['format']} - {f['url']}")

        f_video = sorted(fs, key=lambda f:(f.get('height', 0), f.get('vbr', 0)))[-1]
        print_('video: {}'.format(f_video['format']))

        if f_video.get('abr'):
            f_audio = None
        else:
            fs_audio = sorted([f_audio for f_audio in fs if f_audio.get('abr')], key=lambda f:f['abr'])
            if fs_audio:
                f_audio = fs_audio[-1]
            else:
                raise Exception('no audio')
        print_('audio: {}'.format(f_audio['format']))

        title = info['title']
        url_thumb = info['thumbnail']
        ext = info['ext']

        session.headers.update(info.get('http_headers', {}))

        mobj = re.match(_VALID_URL, self['referer'])
        video_id = mobj.group('id')

        info = {
            'url': f_video['url'],
            'url_thumb': url_thumb,
            'name': format_filename(title, video_id, ext),
            }

        if f_audio:
            def f():
                audio = f_audio['url']
                path = os.path.join(DIR, f'{uuid()}_a.tmp')
                if cw is not None:
                    cw.trash_can.append(path)
                if constants.FAST:
                    downloader_v3.download(audio, session=self.session, chunk=1024*1024, n_threads=2, outdir=os.path.dirname(path), fileName=os.path.basename(path), customWidget=cw, overwrite=True)
                else:
                    downloader.download(audio, session=self.session, outdir=os.path.dirname(path), fileName=os.path.basename(path), customWidget=cw, overwrite=True)
                self.audio_path = path
                print_('audio done')
            self.thread_audio = threading.Thread(target=f, daemon=True)
            self.thread_audio.start()

        return info

    def pp(self, filename):
        if self.thread_audio:
            self.thread_audio.join()
            ffmpeg.merge(filename, self.audio_path, cw=self.cw)
        return filename


# 1804
@try_n(2)
def fix_url(url, cw=None):
    print_ = get_print(cw)
    if '?' in url:
        tail = url.split('?')[1]
    else:
        tail = None
    soup = downloader.read_soup(url, methods={'requests'})
    err = soup.find('div', class_='error-text')
    if err:
        raise errors.Invalid('{}: {}'.format(err.text.strip(), url))
    meta = soup.find('meta', {'itemprop': 'url'})
    if meta:
        url_new = meta.attrs['content']
        if tail:
            url_new = '{}?{}'.format(url_new, tail)
        print_('redirect: {} -> {}'.format(url, url_new))
    else:
        url_new = url
        print_('no redirect')
    return url_new



class Downloader_bili(Downloader):
    type = 'bili'
    URLS = [r'regex:'+_VALID_URL, 'space.bilibili.com/']
    lock = True
    detect_removed = False
    detect_local_lazy = False
    display_name = 'bilibili'
    single = True
    ACCEPT_COOKIES = [r'(.*\.)?bilibili\.com']

    def init(self):
        self.url = fix_url(self.url, self.cw)
        if 'bilibili.com' not in self.url.lower():
            self.url = 'https://www.bilibili.com/video/{}'.format(self.url)
        self.url = self.url.replace('m.bilibili', 'bilibili')
        self.session = Session()

    @classmethod
    def key_id(cls, url):
        mobj = re.match(_VALID_URL, url)
        video_id = mobj.group('id')
        qs = query_url(url)
        p = qs.get('p', ['1'])[0] #6580
        return f'{video_id or url} {p}'

    @property
    def id_(self):
        mobj = re.match(_VALID_URL, self.url)
        video_id = mobj.group('id')
        #anime_id = mobj.group('anime_id')
        return video_id

    def read(self):
        sd = self.session.cookies.get('SESSDATA', domain='.bilibili.com')
        self.print_('sd: {}'.format(sd))
        if not sd: #5647
            self.cw.showCookie()
            self.cw.showLogin('https://passport.bilibili.com/login', 1030, None)

        sid = re.find(r'/channel/collectiondetail?sid=([0-9]+)', self.url)
        mid = re.find(r'space.bilibili.com/([0-9]+)', self.url)
        if sid or mid:
            if not sd:
                raise errors.LoginRequired()
            if sid:
                url_api = f'https://api.bilibili.com/x/polymer/web-space/seasons_archives_list?mid={mid}&season_id={sid}'
                j = downloader.read_json(url_api, self.url)
                title = clean_title(j['data']['meta']['name'])
            elif mid:
                url_api = f'https://api.bilibili.com/x/space/wbi/acc/info?mid={mid}'
                j = downloader.read_json(url_api, self.url)
                title = clean_title(j['data']['name'])
            else:
                raise NotImplementedError()
            self.single = False
            options = {
                'extract_flat': True,
                'playlistend': get_max_range(self.cw),
                }
            ydl = ytdl.YoutubeDL(options, cw=self.cw)
            info = ydl.extract_info(self.url)
            files = []
            for e in info['entries']:
                files.append(File_bili({'referer': e['url']}))
            self.print_(f'urls: {len(files)}')
            file = self.process_playlist(title, files)
            self.title = title
        else:
            file = File_bili({'referer': self.url})
            file.ready(self.cw)
            self.urls.append(file)
            self.title = os.path.splitext(file['name'])[0]

        thumb = BytesIO()
        downloader.download(file['url_thumb'], buffer=thumb)
        self.setIcon(thumb)
        n = int(math.ceil(8.0 / len(self.urls)))
        self.print_(f'n_threads: {n}')
        self.enableSegment(n_threads=n, overwrite=True)



File: src/extractor/tumblr_downloader.py

#coding:utf8
import downloader
from translator import tr_
from utils import Session, query_url, get_max_range, Downloader, clean_title, update_url_query, get_print, get_ext, LazyUrl, urljoin, check_alive, limits
import ree as re
import errors
from error_printer import print_error


class Image:

    def __init__(self, url, id, referer, p, cw=None):
        self._url = url
        self.id_ = id
        self.p = p
        self.cw = cw
        self.url = LazyUrl(referer, self.get, self)

    @limits(.25)
    def get(self, _):
        print_ = get_print(self.cw)
        url = self._url
        ext = get_ext(url)
        if ext.lower()[1:] not in ['jpg', 'png', 'mp4']: #4645
            print_('get_ext: {}, {}'.format(self.id_, url))
            try:
                ext = downloader.get_ext(url, referer=_)
            except Exception as e: #3235
                print_('Err: {}, {}\n'.format(self.id_, url)+print_error(e))
        self.filename = '{}_p{}{}'.format(self.id_, self.p, ext)
        return url



class Downloader_tumblr(Downloader):
    type = 'tumblr'
    URLS = ['tumblr.com']
    MAX_CORE = 4
    ACCEPT_COOKIES = [r'(.*\.)?tumblr\.com']

    def init(self):
        if 'tumblr.com/post/' in self.url:
            raise errors.Invalid(tr_('개별 다운로드는 지원하지 않습니다: {}').format(self.url))
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        qs = query_url(url)
        path = qs.get('redirect_to')
        if path:
            url = urljoin('https://tumblr.com', path[0])
        id = get_id(url)
        return 'https://{}.tumblr.com'.format(id)

    def read(self):
        username = get_id(self.url)
        name = get_name(username, self.session)

        for img in get_imgs(username, self.session, cw=self.cw):
            self.urls.append(img.url)

        self.title = clean_title('{} (tumblr_{})'.format(name, username))



class TumblrAPI:
    _url_base = 'https://www.tumblr.com/api'
    _hdr = {
        'referer': 'https://www.tumblr.com',
        'authorization': 'Bearer aIcXSOoTtqrzR8L8YEIOmBeW94c3FmbSNSWAUbxsny9KKx5VFh',
        }
    _qs = {
    'fields[blogs]': 'name,avatar,title,url,is_adult,?is_member,description_npf,uuid,can_be_followed,?followed,?advertiser_name,is_paywall_on,theme,subscription_plan,?primary,share_likes,share_following,can_subscribe,subscribed,ask,?can_submit,?is_blocked_from_primary,?tweet,?admin,can_message,?analytics_url,?top_tags,paywall_access',
    'npf': 'true',
    'reblog_info': 'false',
    'include_pinned_posts': 'false',
    #'page_number': None,
    }

    def __init__(self, session, cw=None):
        self.session = session
        self.cw = cw

    def print_(self, s):
        get_print(self.cw)(s)

    @limits(1)
    def call(self, path, qs, default_qs=True):
        if default_qs:
            qs_new = qs
            qs = self._qs.copy()
            qs.update(qs_new)
        url = self._url_base + path
        url = update_url_query(url, qs)
        r = self.session.get(url, headers=self._hdr)
        data = r.json()
        errs = data.get('errors', [])
        if errs:
            code = int(errs[0]['code'])
            if code == 0:
                raise Exception('Not found')
            elif code == 4012:
                raise errors.LoginRequired(errs[0]['detail'])
        r.raise_for_status()
        return data['response']

    def name(self, username):
        path = '/v2/blog/{}/posts'.format(username)
        data = self.call(path, {})
        return data['blog']['title'] or data['blog']['name']

    def posts(self, username):
        path = '/v2/blog/{}/posts'.format(username)
        qs = {}
        ids = set()
        default_qs = True
        while True:
            check_alive(self.cw)
            data = self.call(path, qs, default_qs=default_qs)
            for post in (post for post in data['posts'] if post['object_type'] != 'backfill_ad'):
                id_ = post['id']
                if id_ in ids:
                    self.print_('duplicate: {}'.format(id_))
                    continue
                ids.add(id_)
                url = 'https://{}.tumblr.com/post/{}'.format(username, id_)
                yield Post(post, url, self.cw)
            try:
                links = data.get('links') or data['_links']
                path_next = links['next']['href']
            except:
                path_next = None
            if path_next:
                path = path_next
                default_qs = False
            else:
                break


class Post:

    def __init__(self, data, url, cw=None):
        id_ = data['id']
        self.imgs = []

        cs = data['content']
        for trail in data['trail']:
            cs += trail['content']

        for c in cs:
            if c['type'] in ['image', 'video']:
                media = c.get('media')
                if not media: #2859
                    continue
                if isinstance(media, list):
                    media = media[0]
                img = media['url']
                self.imgs.append(Image(img, id_, url, len(self.imgs), cw))
            elif c['type'] in ['text', 'link', 'audio']:
                continue
            else:
                raise NotImplementedError(id_, c)



def get_name(username, session):
    return TumblrAPI(session).name(username)


def get_imgs(username, session, cw=None):
    artist = get_name(username, session)
    imgs = []
    max_pid = get_max_range(cw)
    api = TumblrAPI(session, cw)
    for post in api.posts(username):
        check_alive(cw)
        imgs += post.imgs

        s = '{}  {} (tumblr_{}) - {}'.format(tr_('읽는 중...'), artist, username, len(imgs))
        if cw:
            cw.setTitle(s)
        else:
            print(s)

        if len(imgs) > max_pid:
            break

    return imgs[:max_pid]


def get_id(url):
    if '/dashboard/blog/' in url:
        url = re.find('/dashboard/blog/([0-9a-zA-Z_-]+)', url)
    if '/login_required/' in url:
        url = url.split('/login_required/')[1].split('?')[0].split('/')[0]
    if 'tumblr.com/blog/view/' in url:
        url = url.split('tumblr.com/blog/view/')[1]
    if 'tumblr.com' in url:
        qs = query_url(url)
        url_ = qs.get('url')
        if url_:
            return get_id(url_)
        id = url.split('tumblr.com')[0].split('/')[-1].strip('.')
        if id == 'www':
            url = re.find(r'tumblr\.com/([^/#?]+)', url, err='no id') #6333
        else:
            url = id
    return url



File: src/extractor/artstation_downloader.py

#coding:utf8
import downloader
from error_printer import print_error
from translator import tr_
from utils import Downloader, Soup, get_print, lazy, Session, try_n, File, clean_title, check_alive, get_ext, get_max_range
import dateutil.parser
import utils


class File_artstation(File):
    type = 'artstation'
    format = '[date] name_ppage'
    c_alter = 0

    def alter(self): #6401
        self.c_alter += 1
        if self.c_alter % 2 == 0:
            url = self['url']
        else:
            url = self['url'].replace('/4k/', '/large/')
        return url



class Downloader_artstation(Downloader):
    type = 'artstation'
    URLS = ['artstation.com']
    display_name = 'ArtStation'
    ACCEPT_COOKIES = [r'(.*\.)?artstation\.(com|co)']
    url_main = None

    @try_n(8)
    def init(self):
        # 3849
        self.session = Session()

        import clf2
        clf2.solve(self.url, self.session, self.cw)

        _ = self._id.replace('artstation_', '', 1)
        self.url_main = f'https://www.artstation.com/{_}'

        if '/artwork/' in self.url or '/projects/' in self.url:
            pass
        else:
            self.url = self.url_main
        self.print_(self.url)

    @classmethod
    def fix_url(cls, url): #6516
        if '.artstation.com' in url:
            sub = url.split('.artstation.com')[0].split('/')[-1]
            if sub != 'www':
                url = f'https://www.artstation.com/{sub}'
        return url

    @lazy
    def _id(self):
        _id = get_id(self.url, self.cw)
        return f'artstation_{_id}'

    @lazy
    @try_n(2)
    def name(self):
        soup = downloader.read_soup(self.url_main, session=self.session)
        name = soup.find('meta', {'property': 'og:title'}).attrs['content']
        return clean_title(f'{name} ({self._id})')

    def read(self):
        self.title = self.name
        id_ = self._id.replace('artstation_', '', 1)
        if '/' in id_:
            id_ = id_.split('/')[0]

        if '/artwork/' in self.url or '/projects/' in self.url:
            id_art = get_id_art(self.url)
            imgs = get_imgs_page(id_art, self.session, cw=self.cw)
        else:
            imgs = get_imgs(id_, self.title, self.session, cw=self.cw)

        for img in imgs:
            self.urls.append(img)

        self.title = self.name


@try_n(2)
def get_imgs(id_, title, session, cw=None):
    print_ = get_print(cw)
    referer = f'https://www.artstation.com/{id_}'
    downloader.read_html(referer, session=session)
    #print(session.cookies.keys())

    url = f'https://www.artstation.com/users/{id_}/quick.json'
    j = downloader.read_json(url, referer, session=session)
    uid = j['id']

    datas = []
    ids = set()
    for p in range(1, 1000):
        check_alive(cw)
        url = f'https://www.artstation.com/users/{id_}/projects.json??user_id={uid}&page={p}' #6516
        j = try_n(4)(downloader.read_json)(url, referer, session=session)

        data = j['data']
        if not data:
            break
        for d in data:
            if d['id'] not in ids:
                ids.add(d['id'])
                datas.append(d)
        if cw:
            cw.setTitle(f'{tr_("페이지 읽는 중...")}  {title} - {len(datas)}')
        else:
            print(len(datas))

    datas = sorted(datas, key=lambda data: int(data['id']), reverse=True)

    imgs = []
    i = 0
    names = set()
    while i < len(datas):
        check_alive(cw)
        data = datas[i]
        date = data['created_at']
        post_url = data['permalink']
        #print('post_url', post_url)
        id_art = get_id_art(post_url)
        imgs += get_imgs_page(id_art, session, date=date, cw=cw, names=names)
        if len(imgs) >= get_max_range(cw):
            break
        if cw:
            cw.setTitle(f'{tr_("이미지 읽는 중...")}  {title} - {i+1} / {len(datas)}  ({len(imgs)})')
        else:
            print(len(imgs))
        i += 1

    return imgs


def get_id_art(post_url):
    return post_url.split('/artwork/')[-1].split('/projects/')[-1].split('/')[0].split('?')[0].split('#')[0]


def get_id(url, cw=None):
    print_ = get_print(cw)

    url = url.split('?')[0].split('#')[0]

    if '/artwork/' in url:
        id_art = get_id_art(url)
        imgs = get_imgs_page(id_art, session=Session(), cw=cw)
        return imgs[0].data['user']['username']

    if '.artstation.' in url and 'www.artstation.' not in url:
        id_ = url.split('.artstation')[0].split('//')[-1]
        type_ = None
    elif 'artstation.com' in url:
        paths = url.split('artstation.com/')[1].split('/')
        id_ = paths[0]
        type_ = paths[1] if len(paths) > 1 else None
    else:
        id_ = url.replace('artstation_', '').replace('／', '/')
        type_ = None

    if type_ not in [None, 'likes']:
        type_ = None

    print_(f'type: {type_}, id: {id_}')

    if type_:
        return f'{id_}/{type_}'
    return id_


def get_imgs_page(id_art, session, date=None, cw=None, names=None):
    print_ = get_print(cw)
    url_json = f'https://www.artstation.com/projects/{id_art}.json'
    post_url = f'https://www.artstation.com/artwork/{id_art}'

    name = post_url.strip('/').split('/')[-1]
    if names is not None:
        while name.lower() in names:
            name += '_'
        names.add(name.lower())

    try:
        data = downloader.read_json(url_json, session=session, referer=post_url)
        imgs_ = data['assets']
    except Exception as e:
        print_(print_error(e))
        return []

    if date is None:
        date = data['created_at']
    date = dateutil.parser.parse(date)

    imgs = []
    for page, img in enumerate(imgs_):
        if not img['has_image']:
            print('no img')
            continue
        url = None
        embed = img.get('player_embedded')
        if embed:
            soup = Soup(embed)
            url_embed = soup.find('iframe').attrs['src']
            print_(f'embed: {url_embed}')
            try:
                soup = downloader.read_soup(url_embed, post_url, session=session)
                v = soup.find('video')
                if v:
                    url = v.find('source').attrs['src']
            except Exception as e:
                print_(print_error(e))
            if not url:
                try:
                    url = soup.find('link', {'rel': 'canonical'}).attrs['href']
                    print_(f'YouTube: {url}')
                    raise Exception('YouTube')
                except Exception as e:
                    print(e)
                    url = None
        if not url:
            url = img['image_url']

        d = {
            'date': date,
            'name': clean_title(name),
            'page': page,
            }
        filename = utils.format('artstation', d, get_ext(url))
        img = File_artstation({'referer':post_url, 'url':url.replace('/large/', '/4k/'), 'name': filename})
        img.data = data

        imgs.append(img)

    return imgs



File: src/extractor/coub_downloader.py

from utils import Downloader, LazyUrl, try_n, format_filename, get_ext
import ytdl
from io import BytesIO as IO
import downloader
import ree as re
PATTEN_IMAGIZER = r'coub-com-.+\.imagizer\.com'


def get_id(url):
    return re.find(r'/view/([0-9a-z]+)', url, err='no id')



class Downloader_coub(Downloader):
    type = 'coub'
    URLS = ['coub.com', r'regex:'+PATTEN_IMAGIZER]
    single = True
    ACCEPT_COOKIES = [r'(.*\.)?coub\.com']

    @classmethod
    def fix_url(cls, url):
        return re.sub(PATTEN_IMAGIZER, 'coub.com', url)

    @classmethod
    def key_id(cls, url):
        return get_id(url)

    def read(self):
        video = Video(self.url, cw=self.cw)
        video.url()#

        self.urls.append(video.url)
        self.setIcon(video.thumb)

        self.enableSegment()

        self.title = video.title



class Video:
    _url = None

    def __init__(self, url, cw=None):
        self.url = LazyUrl(url, self.get, self, pp=self.pp)
        self.cw = cw

    @try_n(2)
    def get(self,  url):
        if self._url:
            return self._url

        ydl = ytdl.YoutubeDL(cw=self.cw)
        info = ydl.extract_info(url)
        fs = [f for f in info['formats'] if f['ext'] == 'mp4']
        f = sorted(fs, key=lambda f: int(f.get('filesize', 0)))[-1]
        self._url = f['url']
##        fs = [f for f in info['formats'] if f['ext'] == 'mp3']
##        self.f_audio = sorted(fs, key=lambda f: int(f.get('filesize', 0)))[-1]

        self.thumb_url = info['thumbnails'][0]['url']
        self.thumb = IO()
        downloader.download(self.thumb_url, buffer=self.thumb)
        self.title = info['title']
        ext = get_ext(self._url)
        self.filename = format_filename(self.title, info['id'], ext)
        return self._url

    def pp(self, filename):
##        import ffmpeg
##        f = IO()
##        downloader.download(self.f_audio['url'], buffer=f)
##        ffmpeg.merge(filename, f)
        return filename



File: src/extractor/etc_downloader.py

import downloader
import ytdl
from utils import Downloader, Session, try_n, LazyUrl, get_ext, format_filename, get_print, get_resolution
from io import BytesIO
import ree as re
from m3u8_tools import playlist2stream, M3u8_stream
import utils
import ffmpeg
import clf2
import os



class Downloader_etc(Downloader):
    type = 'etc'
    URLS = ['thisvid.com'] #5153
    single = True
    MAX_PARALLEL = 8
    display_name = 'Etc'
    PRIORITY = 10

    def init(self):
        self.session = Session()
        name = ytdl.get_extractor_name(self.url)
        self.print_('extractor: {}'.format(name))
        if name == 'ixigua': #6290
            clf2.solve(self.url, session=self.session)
        #if name == 'generic':
        #    raise NotImplementedError()

    def read(self):
        video = get_video(self.url, self.session, self.cw)

        if video.artist:
            self.artist = video.artist

        self.urls.append(video.url)

        self.print_('url_thumb: {}'.format(video.url_thumb))
        self.setIcon(video.thumb)
        if video.header.lower() not in ['yourporn', 'spankbang']:
            self.enableSegment()#
        if isinstance(video.url(), M3u8_stream):
            self.disableSegment()

        self.title = os.path.splitext(video.filename)[0].replace('：', ':')


def int_or_none(s):
    try:
        return int(s)
    except:
        return None


def format_(f):
    if f is None:
        return 'None'
    return 'format:{} - resolution:{} - vbr:{} - audio:{} - url:{}'.format(f['format'], f['_resolution'], f['_vbr'], f['_audio'], f['url'])


class UnSupportedError(Exception):pass


def get_video(url, session, cw, ie_key=None):
    print_ = get_print(cw)
    try:
        video = _get_video(url, session, cw, ie_key, allow_m3u8=True)
        if isinstance(video, Exception):
            raise video
        if isinstance(video.url(), M3u8_stream):
            c = video.url().segs[0].download(2, cw)
            if not c:
                raise Exception('invalid m3u8')
        return video
    except Exception as e:
        if isinstance(e, UnSupportedError):
            raise e
        print_(e)
        return _get_video(url, session, cw, ie_key, allow_m3u8=False)

@try_n(4)
def _get_video(url, session, cw, ie_key=None, allow_m3u8=True):
    print_ = get_print(cw)
    print_('get_video: {}, {}'.format(allow_m3u8, url))
    options = {
        'noplaylist': True,
        #'extract_flat': True,
        'playlistend': 1,
        'writesubtitles': True,
        }
    if ytdl.get_extractor_name(url) == 'spankbang':
        options['legacyserverconnect'] = True #6545
    ydl = ytdl.YoutubeDL(options, cw=cw)
    try:
        info = ydl.extract_info(url)
    except Exception as e:
        if 'ERROR: Unsupported URL' in str(e):
            return UnSupportedError(str(e))
        raise e
    if not ie_key:
        ie_key = ytdl.get_extractor_name(url)
    info['ie_key'] = ie_key
    url_new = info.get('url')
    formats = info.get('formats', [])

    if not formats and (info.get('entries') or 'title' not in info):
        if 'entries' in info:
            entry = info['entries'][0]
            url_new = entry.get('url') or entry['webpage_url']
        if url_new != url:
            return get_video(url_new, session, cw, ie_key=get_ie_key(info))

    session.headers.update(info.get('http_headers', {}))
    #session.cookies.update(ydl.cookiejar)

    if not formats:
        if url_new:
            f = {'url': url_new, 'format': ''}
            formats.append(f)

    fs = []
    for i, f in enumerate(formats):
        f['_index'] = i
        f['_resolution'] = int_or_none(re.find(r'([0-9]+)p', f['format'], re.I)) or f.get('height') or f.get('width') or int_or_none(f.get('quality')) or int(f.get('vcodec', 'none') != 'none') #5995
        f['_vbr'] = f.get('vbr') or 0
        f['_audio'] = f.get('abr') or f.get('asr') or int(f.get('acodec', 'none') != 'none')
        print_(format_(f))
        fs.append(f)

    #4773
    res = max(get_resolution(), min(f['_resolution'] for f in fs))
    print_(f'res: {res}')
    fs = [f for f in fs if f['_resolution'] <= res]

    if not fs:
        raise Exception('No videos')

    def filter_f(fs):
        for f in fs:
            if allow_m3u8:
                return f
            ext = get_ext_(f['url'], session, url)
            if ext.lower() != '.m3u8':
                return f
            print_('invalid url: {}'.format(f['url']))
        return list(fs)[0]#

    f_video = filter_f(sorted(fs, key=lambda f:(f['_resolution'], int(bool(f['_audio'])), f['_vbr'], f['_index']), reverse=True)) #6072, #6118
    print_('video0: {}'.format(format_(f_video)))

    if f_video['_audio']:
        f_audio = None
    else:
        fs_audio = sorted([f_audio for f_audio in fs if (not f_audio['_resolution'] and f_audio['_audio'])], key=lambda f:(f['_audio'], f['_vbr'], f['_index']))
        if fs_audio:
            f_audio = fs_audio[-1]
        else:
            try:
                print_('trying to get f_video with audio')
                f_video = filter_f(reversed(sorted([f for f in fs if f['_audio']], key=lambda f:(f['_resolution'], f['_index']))))
            except Exception as e:
                print_('failed to get f_video with audio: {}'.format(e))
            f_audio = None
    print_('video: {}'.format(format_(f_video)))
    print_('audio: {}'.format(format_(f_audio)))
    video = Video(f_video, f_audio, info, session, url, cw=cw)

    return video


def get_ie_key(info):
    ie_key = info.get('ie_key') or info['extractor']
    ie_key = ie_key.split(':')[0]
    if ie_key.lower().endswith('playlist'):
        ie_key = ie_key[:-len('playlist')]
    return ie_key


def get_ext_(url, session, referer):
    try:
        ext = downloader.get_ext(url, session, referer)
    except Exception as e:
        ext = get_ext(url)
    return ext


class Video:
    live = False

    def __init__(self, f, f_audio, info, session, referer, cw=None):
        self.f_audio = f_audio
        self.cw = cw
        self.title = title = info['title']
        self.id = info['id']
        self.url = f['url']
        self.artist = info.get('uploader')
        self.header = utils.capitalize(get_ie_key(info))
        self.session = session
        self.referer = referer
        self.subs = ytdl.get_subtitles(info)

        self.url_thumb = info.get('thumbnail')
        self.thumb = BytesIO()
        if self.url_thumb:
            downloader.download(self.url_thumb, referer=referer, buffer=self.thumb, session=session)

        ext = get_ext_(self.url, session, referer)

        def foo():
            hdr = session.headers.copy()
            if referer:
                hdr['Referer'] = referer
            self.live = True
            return utils.LiveStream(self.url, headers=hdr, fragments=f.get('fragments') if ytdl.LIVE_FROM_START.get('etc') else None)

        if not ext:
            if f['_resolution']:
                ext = '.mp4'
            else:
                ext = '.mp3'

        if ext.lower() == '.m3u8':
            res = get_resolution() #4773
            if info.get('live_status') == 'is_live':
                url = foo()
            else:
                try:
                    url = playlist2stream(self.url, referer, session=session, n_thread=4)
                except:
                    url = M3u8_stream(self.url, referer=referer, session=session, n_thread=4)
                if url.live is not None: #5110
                    url = foo()
            ext = '.mp4'
        elif ext.lower() == '.mpd': # TVer
            url = foo()
            ext = '.mp4'
        else:
            url = self.url
        self.url = LazyUrl(referer, lambda x: url, self, pp=self.pp)
        info_ext = info.get('ext')
        if info_ext == 'unknown_video': #vk
            info_ext = None
        self.filename = format_filename(title, self.id, info_ext or ext, header=self.header, live=self.live)

    def pp(self, filename):
        if self.f_audio:
            f = BytesIO()
            downloader.download(self.f_audio['url'], buffer=f, referer=self.referer, session=self.session)
            ffmpeg.merge(filename, f, cw=self.cw)
        utils.pp_subtitle(self, filename, self.cw)
        return filename



File: src/extractor/_4chan_downloader.py

import downloader
from utils import Downloader, File, clean_title, urljoin, get_ext, limits
import utils



class File_4chan(File):
    type = '4chan'
    format = 'page:04;'

    @limits(.5)
    def get(self):
        return {}



class Downloader_4chan(Downloader):
    type = '4chan'
    URLS = [r'regex:boards.(4chan|4channel).org']
    MAX_CORE = 4
    display_name = '4chan'
    ACCEPT_COOKIES = [r'(.*\.)?(4chan|4channel)\.org']

    @classmethod
    def fix_url(cls, url):
        return url.split('#')[0]

    def read(self):
        soup = downloader.read_soup(self.url)
        for div in soup.findAll('div', class_='fileText'):
            href = urljoin(self.url, div.a['href'])
            d = {
                'page': len(self.urls),
                }
            file = File_4chan({'url': href, 'referer': self.url, 'name': utils.format('4chan', d, get_ext(href))})
            self.urls.append(file)

        board = self.url.split('/')[3]
        title = soup.find('span', class_='subject').text
        id_ = int(self.url.split('/thread/')[1].split('/')[0])
        self.title = clean_title(f'[{board}] {title} ({id_})')



File: src/extractor/pornhub_downloader.py

#coding:utf8
'''
Pornhub Downloader
'''
from io import BytesIO
import downloader
import ree as re
from utils import (Downloader, Soup, try_n, LazyUrl, urljoin, get_print,
                   Session, get_max_range, filter_range, get_ext,
                   format_filename, clean_title, get_resolution, check_alive)
import clf2
import utils
from m3u8_tools import playlist2stream, M3u8_stream
import errors
from error_printer import print_error
import ytdl



class File:
    '''
    File
    '''
    _thumb = None

    def __init__(self, id_, title, url, url_thumb, artist=''):
        self.id_ = id_
        self.title = clean_title(f'{title}')
        self.url = url

        ext = get_ext(self.url)
        if ext.lower() == '.m3u8':
            try:
                self.url = playlist2stream(self.url, n_thread=4)
            except:
                self.url = M3u8_stream(self.url, n_thread=4)

        self.url_thumb = url_thumb

        if ext.lower() == '.m3u8':
            ext = '.mp4'
        self.filename = format_filename(self.title, self.id_, ext, artist=artist)

    def thumb(self):
        if self._thumb is None:
            f = BytesIO()
            downloader.download(self.url_thumb, buffer=f)
            self._thumb = f
        else:
            f = self._thumb
        f.seek(0)
        return f


class Video:
    '''
    Video
    '''
    _url = None
    filename = None
    thumb = None

    def __init__(self, url, cw, session):
        url = Downloader_pornhub.fix_url(url)
        self.url = LazyUrl(url, self.get, self)
        self.cw = cw
        self.session = session

    @try_n(2)
    def get(self, url):
        '''
        get
        '''
        cw = self.cw
        session = self.session
        print_ = get_print(cw)
        if self._url:
            return self._url

        id_ = re.find(r'viewkey=(\w+)', url, re.I) or \
              re.find(r'/embed/(\w+)', url, re.I)
        print_('id: {}'.format(id_))
        if 'viewkey=' not in url.lower() and '/gif/' not in url.lower():
            if id_ is None:
                raise Exception('no id')
            url = urljoin(url, '/view_video.php?viewkey={}'.format(id_))

        url_test = url.replace('pornhubpremium.com', 'pornhub.com')
        try:
            html = downloader.read_html(url_test, session=session)
            soup = Soup(html)
            if soup.find('div', id='lockedPlayer'):
                print_('Locked player')
                raise Exception('Locked player')
            url = url_test
        except Exception as e: #3511
            print_(print_error(e))
            url = url.replace('pornhub.com', 'pornhubpremium.com')
            html = downloader.read_html(url, session=session)

        soup = Soup(html)
        soup = fix_soup(soup, url, session, cw)
        html = soup.html

        # removed
        if soup.find('div', class_='removed'):
            raise Exception('removed')

        gif = soup.find('div', {'id': 'gifImageSection'})
        if gif:
            print_('GIF')
            id_ = url.split('/gif/')[1]
            id_ = re.findall('[0-9a-zA-Z]+', id_)[0]

            jss = list(gif.children)
            for js in jss:
                if 'data-mp4' in getattr(js, 'attrs', {}):
                    break
            else:
                raise Exception('gif mp4 url not found')

            title = js['data-gif-title']
            url = js['data-mp4']
            url_thumb = re.find(r'https?://.+?.phncdn.com/pics/gifs/.+?\.jpg', html, err='no thumb')
            file = File('gif_{}'.format(id_), title, url, url_thumb)
        else:
            if id_ is None:
                raise Exception('no id')

            print_('Video')

            # 1968
            title = soup.find('h1', class_='title')
            for item in title.findAll(class_='phpFree'):
                item.decompose()
            title = title.text.strip()

            #4940
            artist = soup.find('div', class_='userInfo').find('div', class_='usernameWrap').text.strip()

            ydl = ytdl.YoutubeDL(cw=cw)
            info = ydl.extract_info(url)
            session.headers.update(info.get('http_headers', {}))

            fs = []
            for f in info['formats']:
                f['quality'] = f.get('height') or 0
                if f['protocol'].startswith('m3u8'):
                    f['quality'] -= 1
                if 'dash' in f['protocol'].lower(): #5554
                    continue
                print_('[{}p] {} {}'.format(f['height'], f['protocol'], f['url']))
                fs.append(f)

            if not fs:
                raise Exception('No formats')

            fs = sorted(fs, key=lambda f: f['quality'])

            res = get_resolution()

            fs_good = [f for f in fs if f['quality'] <= res]
            if fs_good:
                f = fs_good[-1]
            else:
                f = fs[0]
            print_('\n[{}p] {} {}'.format(f['height'], f['protocol'], f['url']))

            file = File(id_, title, f['url'], info['thumbnail'], artist)

        self._url = file.url
        self.title = file.title
        self.filename = file.filename
        self.thumb = file.thumb
        return self._url


def is_login(session, cw=None, n=2):
    '''
    is_login
    '''
    print_ = get_print(cw)
    print_('is_login {}'.format(n))
    if n <= 0:
        return False
    url = 'https://www.pornhubpremium.com'
    soup = downloader.read_soup(url, session=session)
    soup = fix_soup(soup, url, session, cw)
    if soup.find('ul', id='profileMenuDropdown'):
        return True
    return is_login(session, cw, n-1)



class Downloader_pornhub(Downloader):
    '''
    Downloader
    '''
    type = 'pornhub'
    single = True
    strip_header = False
    URLS = ['pornhub.com', 'pornhubpremium.com', 'pornhubthbh7ap3u.onion']
    ACCEPT_COOKIES = [r'.*(pornhub|phncdn|pornhubpremium).*'] #6181

    @classmethod
    def fix_url(cls, url):
        if 'pornhub_gif_' in url:
            url = 'https://www.pornhub.com/gif/{}'.format(
                url.replace('pornhub_gif_', ''))
        elif 'pornhub_album_' in url:
            url = 'https://www.pornhub.com/album/{}'.format(
                url.replace('pornhub_album_', ''))
        elif 'pornhub_' in url:
            url = 'https://www.pornhub.com/view_video.php?viewkey={}'\
                       .format(url.replace('pornhub_', ''))
        if '/authenticate/goToLoggedIn' in url:
            qs = utils.query_url(url)
            url = urljoin(url, qs['url'][0])
        url = url.replace('pornhubthbh7ap3u.onion', 'pornhub.com')
        return url

    @classmethod
    def key_id(cls, url):
        for domain in cls.URLS:
            if domain in url:
                id_ = domain + url.split(domain)[1]
                break
        else:
            raise Exception('no id')
        return id_.split('#')[0]

    @try_n(2)
    def read(self):
        cw = self.cw

        session = self.session = Session() # 1791
        self.purge_cookies()
        session.cookies.update({
            'age_verified': '1',
            'accessAgeDisclaimerPH': '1',
            'accessPH': '1',
            }) #6124
        if 'pornhubpremium.com' in self.url.lower() and\
           not is_login(session, cw):
            raise errors.LoginRequired(method='browser', url='https://www.pornhubpremium.com/premium/login')

        videos = []
        tab = ''.join(self.url.replace('pornhubpremium.com', 'pornhub.com', 1).split('?')[0].split('#')[0].split('pornhub.com/')[-1].split('/')[2:3])

        if '/album/' in self.url:
            self.print_('Album')
            info = read_album(self.url, session=session)
            self.single = False
            for photo in info['photos']:
                self.urls.append(photo.url)

            self.title = clean_title(info['title'])
        elif '/photo/' in self.url:
            self.print_('Photo')
            info = read_photo(self.url, session=session)
            self.urls.append(info['photo'].url)

            self.title = info['title']
        elif tab not in ['', 'videos']:
            raise NotImplementedError(tab)
        elif 'viewkey=' not in self.url.lower() and\
             '/embed/' not in self.url.lower() and\
             '/gif/' not in self.url.lower():
            self.print_('videos')
            info = get_videos(self.url, session, cw)
            hrefs = info['hrefs']
            self.print_('videos: {}'.format(len(hrefs)))

            if not hrefs:
                raise Exception('no hrefs')

            videos = [Video(href, cw, session) for href in hrefs]
            video = self.process_playlist(info['title'], videos)
            self.setIcon(video.thumb())
            self.enableSegment()
        else:
            video = Video(self.url, cw, session)
            video.url()
            self.urls.append(video.url)
            self.setIcon(video.thumb())
            self.title = video.title
            self.enableSegment()



def fix_soup(soup, url, session=None, cw=None):
    '''
    fix_soup
    '''
    print_ = get_print(cw)
    if soup.find('div', class_='logo'):
        return soup
    print_('invalid soup: {}'.format(url))

    res = clf2.solve(url, session=session, cw=cw)
    session.purge(Downloader_pornhub.ACCEPT_COOKIES)

    return Soup(res['html'])



class Photo_lazy:
    '''
    Photo_lazy
    '''

    def __init__(self, url, session):
        self._session = session
        self.url = LazyUrl(url, self.get, self)

    def get(self, url):
        info = read_photo(url, self._session)
        photo = info['photo']
        url = photo.url()
        self.filename = photo.filename
        return url



class Photo:
    '''
    Photo
    '''

    def __init__(self, url, referer, id_, session):
        self._session = session
        ext = get_ext(url)
        self.filename = f'{id_}{ext}'
        self.url = LazyUrl(referer, lambda _: url, self)


@try_n(8)
def read_album(url, session=None):
    '''
    read_album
    '''
    photos = []
    soup = downloader.read_soup(url, session=session)
    id_album = re.find('/album/([0-9]+)', url, err='no album id')
    for block in soup.findAll('div', class_='photoAlbumListBlock'):
        href = block.a.attrs['href']
        href = urljoin(url, href)
        photo = Photo_lazy(href, session)
        photos.append(photo)

    info = {}
    title = soup.find('h1', class_='photoAlbumTitleV2').text
    info['title'] = format_filename(title, f'album_{id_album}')
    info['photos'] = photos
    return info


@try_n(8)
def read_photo(url, session=None):
    '''
    read_photo
    '''
    id_ = re.find('/photo/([0-9]+)', url, err='no photo id')
    soup = downloader.read_soup(url, session=session)
    section = soup.find('div', id='photoImageSection')
    photo = section.find('img')['src']

    info = {}
    info['photo'] = Photo(photo, url, id_, session)
    title = soup.find('h1').text
    info['title'] = format_filename(title, f'photo_{id_}')
    return info


@try_n(4)
def get_videos(url, session, cw=None):
    '''
    get_videos
    '''
    print_ = get_print(cw)

    if '/users/' in url:
        mode = 'users'
        username = url.split('/users/')[1].split('/')[0]
    elif '/pornstar/' in url:
        mode = 'pornstar'
        username = url.split('/pornstar/')[1].split('/')[0]
    elif '/model/' in url:
        mode = 'model'
        username = url.split('/model/')[1].split('/')[0]
    elif '/channels/' in url:
        mode = 'channels'
        username = url.split('/channels/')[1].split('/')[0]
    elif '/playlist/' in url:
        mode = 'playlist'
        username = url.split('/playlist/')[1].split('/')[0]
    else:
        raise Exception('Not supported url')
    username = username.split('?')[0].split('#')[0]

    domain = utils.domain(url)

    if mode in ['pornstar']:
        url_main = 'https://{}/{}/{}'.format(domain, mode, username)
        html = downloader.read_html(url_main, session=session)
        soup = Soup(html)
        soup = fix_soup(soup, url_main, session, cw)
        for a in soup.findAll('a'):
            if '/{}/{}/videos/upload'.format(mode, username) in a.attrs.get('href', ''):
                free = True
                break
        else:
            free = False
        print_('free: {}'.format(free))

    # Range
    max_pid = get_max_range(cw)

    html = downloader.read_html(url, session=session)
    soup = fix_soup(Soup(html), url, session, cw)

    info = {}

    # get title
    h1 = soup.find('h1')
    if h1:
        header = 'Playlist'
        title = h1.parent.find(id='watchPlaylist')
    else:
        title = None
    if not title:
        header = 'Channel'
        profile = soup.find('div', class_='profileUserName')
        wrapper = soup.find('div', class_='titleWrapper')
        bio = soup.find('div', class_='withBio')
        title = soup.find('h1', {'itemprop':'name'})
        if not title and profile:
            title = profile.a
        if not title and wrapper:
            title = wrapper.h1
        if not title and bio:
            title = bio.h1
    if not title:
        raise Exception('No title')
    #print(title)
    info['title'] = '[{}] {}'.format(header, title.text.strip())
    token = re.find('''token *= *['"](.*?)['"]''', html)
    print_('token: {}'.format(token))

    # get links
    hrefs = []
    fail = 0
    for p in range(1, 1+100):
        check_alive(cw)
        try:
            if mode in ['users', 'model']:
                if mode == 'users':
                    url_api = 'https://{}/users/{}/videos/public/'\
                              'ajax?o=mr&page={}'.format(domain, username, p)
                elif mode == 'model':
                    url_api = 'https://{}/model/{}/videos/upload/'\
                              'ajax?o=mr&page={}'.format(domain, username, p)
                r = session.post(url_api)
                soup = Soup(r.text)
                if soup.find('h1'):
                    print('break: h1')
                    break
            elif mode in ['pornstar']:
                if free:
                    url_api = 'https://{}/{}/{}/videos/upload'\
                              '?page={}'.format(domain, mode, username, p)
                    soup = downloader.read_soup(url_api, session=session)
                    soup = fix_soup(soup, url_api, session, cw)
                    soup = soup.find('div', class_='videoUList')
                else:
                    url_api = 'https://{}/{}/{}?page={}'.format(domain, mode, username, p)
                    soup = downloader.read_soup(url_api, session=session)
                    soup = fix_soup(soup, url_api, session, cw)
                    soup = soup.find('ul', class_='pornstarsVideos')
            elif mode in ['channels']:
                url_api = 'https://{}/{}/{}/videos?page={}'.format(domain, mode, username, p)
                soup = downloader.read_soup(url_api, session=session)
                soup = fix_soup(soup, url_api, session, cw)
                try:
                    soup = soup.find('div', {'id': 'channelsBody'}).find('div', class_='rightSide')
                except:
                    break
            elif mode in ['playlist']:
                #url_api = 'https://{}/playlist/viewChunked?id={}&offset={}&itemsPerPage=40'.format(domain, username, len(hrefs))
                if token is None:
                    raise Exception('no token')
                url_api = 'https://{}/playlist/viewChunked?id={}&token={}&page={}'.format(domain, username, token, p)
                soup = downloader.read_soup(url_api, session=session)
            else:
                raise NotImplementedError(mode)
            fail = 0
        except Exception as e:
            print_(e)
            fail += 1
            if fail < 2:
                continue
            else:
                break
        finally:
            print_('{}  ({})'.format(url_api, len(hrefs)))

        lis = soup.findAll('li', class_='videoblock')
        if not lis:
            print_('break: no lis')
            break

        if getattr(soup.find('title'), 'text', '').strip() == 'Page Not Found':
            print_('Page Not Found')
            break

        c = 0
        for li in lis:
            a = li.find('a')
            href = a.attrs['href']
            href = urljoin(url, href)
            if href in hrefs:
                continue
            c += 1
            if href.startswith('javascript:'): # Remove Pornhub Premium
                print(href)
                continue
            hrefs.append(href)
        if c == 0:
            print('c==0')
            break
        print(c) # 1320

        if len(hrefs) >= max_pid:
            break

    if cw:
        hrefs = filter_range(hrefs, cw.range)
        cw.fped = True

    info['hrefs'] = hrefs[:max_pid]

    return info



File: src/extractor/bcy_downloader.py

#coding:utf8
import downloader
from utils import Soup, cut_pair, LazyUrl, Downloader, get_print, get_max_range, try_n, clean_title, check_alive, json
import os
from translator import tr_



class Downloader_bcy(Downloader):
    type = 'bcy'
    URLS = ['bcy.net/item/detail/', 'bcy.net/u/']
    MAX_CORE = 8
    display_name = '半次元'
    ACCEPT_COOKIES = [r'(.*\.)?bcy\.net']

    def init(self):
        self.html = downloader.read_html(self.url)
        self.info = get_info(self.url, self.html)

    @property
    def name(self):
        info = self.info
        if '/detail/' in self.url:
            title = '{} (bcy_{}) - {}'.format(clean_title(info['artist']), info['uid'], info['id'])
        else:
            title = '{} (bcy_{})'.format(clean_title(info['artist']), info['uid'])
        return title

    def read(self):
        imgs = get_imgs(self.url, self.html, cw=self.cw)

        for img in imgs:
            self.urls.append(img.url)

        self.title = self.name
        self.artist = self.info['artist']


def get_ssr_data(html):
    s = html.split('window.__ssr_data = JSON.parse("')[1].replace('\\"', '"')
    s = cut_pair(s).replace('"', '\\"')
    data = json.loads(json.loads('"{}"'.format(s)))
    return data


@try_n(2)
def get_imgs(url, html=None, cw=None):
    if '/detail/' not in url:
        return get_imgs_channel(url, html, cw)

    if html is None:
        html = downloader.read_html(url)

    data = get_ssr_data(html)

    multi = data['detail']['post_data']['multi']

    imgs = []

    for m in multi:
        path = m['original_path']
        img = json.loads('"{}"'.format(path))
        img = Image_single(img, url, len(imgs))
        imgs.append(img)

    return imgs


class Image_single:
    def __init__(self, url ,referer, p):
        self._url = url
        self.p = p
        self.url = LazyUrl(referer, self.get, self)

    def get(self, referer):
        ext = get_ext(self._url, referer)
        self.filename = '{:04}{}'.format(self.p, ext)
        return self._url


class Image:
    def __init__(self, url, referer, id, p):
        self.id = id
        self.p = p
        self._url = url
        self.url = LazyUrl(referer, self.get, self)

    def get(self, referer):
        ext = get_ext(self._url, referer)
        self.filename = '{}_p{}{}'.format(self.id, self.p, ext)
        return self._url


def get_ext(url, referer=None):
    ext = os.path.splitext(url.split('?')[0].replace('~noop.image', ''))[1]
    if ext in ['.image', '']:
        ext = downloader.get_ext(url, referer=referer)
    return ext


def get_info(url, html):
    soup = Soup(html)
    info = {}

    uname = soup.find('div', class_='user-name') or soup.find('p', class_='uname') or soup.find('div', class_='user-info-name')

    info['artist'] = uname.text.strip()

    j = get_ssr_data(html)

    if '/detail/' in url:
        info['uid'] = j['detail']['detail_user']['uid']
        info['id'] = j['detail']['post_data']['item_id']
    else:
        info['uid'] = j['homeInfo']['uid']

    return info


def get_imgs_channel(url, html=None, cw=None):
    print_ = get_print(cw)
    if html is None:
        html = downloader.read_html(url)
    info = get_info(url, html)

    # Range
    max_pid = get_max_range(cw)

    ids = set()
    imgs = []
    for p in range(1000):
        url_api = 'https://bcy.net/apiv3/user/selfPosts?uid={}'.format(info['uid'])
        if imgs:
            url_api += '&since={}'.format(imgs[-1].id)
        data_raw = downloader.read_html(url_api, url)
        data = json.loads(data_raw)['data']
        items = data['items']
        if not items:
            print('no items')
            break
        c = 0
        for item in items:
            check_alive(cw)
            id = item['item_detail']['item_id']
            if id in ids:
                print('duplicate')
                continue
            c += 1
            ids.add(id)
            url_single = 'https://bcy.net/item/detail/{}'.format(id)
            imgs_single = get_imgs(url_single, cw=cw)
            print_(str(id))
            for p, img in enumerate(imgs_single):
                img = Image(img._url, url_single, id, p)
                imgs.append(img)
            s = '{} {} - {}'.format(tr_('읽는 중...'), info['artist'], min(len(imgs), max_pid))
            if cw:
                cw.setTitle(s)
            else:
                print(s)

            if len(imgs) >= max_pid:
                break
        if not c:
            print('not c')
            break
        if len(imgs) >= max_pid:
            print('over max_pid:', max_pid)
            break
    return imgs[:max_pid]



File: src/extractor/nijie_downloader.py

#coding: utf-8
import downloader
from utils import Downloader, Session, urljoin, get_max_range, get_print, clean_title, try_n, get_ext, check_alive, File, limits, clean_url
from translator import tr_
import ree as re
from errors import LoginRequired
import utils


def get_id(url):
    return re.find('id=([0-9]+)', url)


def isLogin(soup):
    if soup.find('ul', id="sub-menu"):
        return True
    return False



class Downloader_nijie(Downloader):
    type = 'nijie'
    URLS = ['nijie.info']
    MAX_CORE = 4
    display_name = 'ニジエ'
    ACCEPT_COOKIES = [r'(.*\.)?nijie\.info']

    def init(self):
        if 'members.php' not in self.url and 'members_illust.php' not in self.url:
            raise NotImplementedError()
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        if 'nijie.info' not in url.lower():
            url = f'https://nijie.info/members.php?id={url}'
        return url.replace('http://', 'https://')

    @property
    def name(self):
        name = self.soup.find('p', class_='user_icon').find('a', class_='name').text.strip()
        name = f'{name} (nijie_{get_id(self.url)})'
        return clean_title(name)

    def read(self):
        id = get_id(self.url)
        self.soup = read_soup(f'https://nijie.info/members.php?id={id}', session=self.session)

        if not isLogin(self.soup):
            raise LoginRequired(method='browser', url='https://nijie.info/login.php')

        self.title = self.name

        self.urls += get_imgs(self.url, self.name, self.session, self.cw)

        self.title = self.name



class Image(File):
    type = 'nijie'

    def get(self):
        url = self['referer']
        if '://' not in url:
            return {'url': url}
        id = int(re.find('[?&]id=([0-9]+)', url))
        url = url.replace('view.php', 'view_popup.php') #6726
        soup = read_soup(url, self['rereferer'], session=self.session)
        view = soup.find('div', id='img_window')
        imgs = []
        p = 0
        for img in view.findAll('img'):
            url_img = urljoin(url, img['src'])
            url_img = re.sub('__rs_l[0-9]+x[0-9]+/', '', url_img)
            if '/filter/' in url_img:
                continue
            ext = get_ext(url_img)
            name = f'{id}_p{p}{ext}'
            imgs.append({'url': url_img, 'name': name})
            p += 1
        return imgs


@try_n(12, sleep=lambda try_: 10+try_*10)
@limits(5)
def read_soup(*args, **kwargs):
    return downloader.read_soup(*args, **kwargs)


def setPage(url, page):
    if 'p=' in url:
        url = re.sub('p=[0-9]*', f'p={page}', url)
    else:
        url += f'&p={page}'

    return url


def get_imgs(url, title=None, session=None, cw=None):
    print_ = get_print(cw)
    url = clean_url(url)

    id = get_id(url)
    url = f'https://nijie.info/members_illust.php?id={id}'

    olds = utils.process_olds(Image, title, r'([0-9]+)_p', cw)
    ids = olds['ids']
    imgs_old = olds['imgs']

    # Range
    max_pid = get_max_range(cw)

    imgs = []
    for p in range(1, 101):
        url = setPage(url, p)
        print_(url)
        soup = read_soup(url, session=session)

        posts = soup.findAll('div', class_='nijie')
        if not posts:
            print_('no posts')
            break

        c = 0
        for post in posts:
            check_alive(cw)
            url_img = urljoin(url, post.a.attrs['href'])
            id_ = int(re.find(r'[&\?]id=([0-9]+)', url_img, err='no id'))
            if id_ in ids:
                continue
            ids.add(id_)
            img = Image({'referer': url_img, 'rereferer': url})

            imgs.append(img)
            c += 1
        print_(f'c: {c}')

        msg = f'{tr_("읽는 중...")}  {title} - {len(imgs)}'
        if cw:
            cw.setTitle(msg)
        else:
            print(msg)

        if len(imgs) >= max_pid or c == 0:
            break
    return imgs + imgs_old



File: src/extractor/rule34_xxx_downloader.py

import downloader
import ree as re
import os
from utils import Downloader, query_url, Soup, get_max_range, get_print, clean_title, try_n, check_alive, clean_url
from translator import tr_
from urllib.parse import quote
LIMIT = 100


def get_tags(url):
    url = clean_url(url)
    qs = query_url(url)
    if 'page=favorites' in url:
        id = qs.get('id', ['N/A'])[0]
        id = 'fav_{}'.format(id)
    else:
        tags = qs.get('tags', [])
        tags.sort()
        id = ' '.join(tags)
    if not id:
        id = 'N/A'
    return id



class Downloader_rule34_xxx(Downloader):
    type = 'rule34_xxx'
    URLS = ['rule34.xxx']
    MAX_CORE = 8
    display_name = 'Rule34.xxx'
    _name = None
    ACCEPT_COOKIES = [r'(.*\.)?rule34\.xxx']

    @classmethod
    def fix_url(cls, url):
        if 'rule34.xxx' in url.lower():
            url = url.replace('http://', 'https://')
        else:
            url = url.replace(' ', '+')
            while '++' in url:
                url = url.replace('++', '+')
            url = quote(url)
            url = url.replace('%2B', '+')
            url = 'https://rule34.xxx/index.php?page=post&s=list&tags={}'.format(url)
        return url

    @property
    def name(self):
        if self._name is None:
            tags = get_tags(self.url)
            self._name = tags
        return clean_title(self._name)

    def read(self):
        self.title = self.name

        imgs = get_imgs(self.url, self.name, cw=self.cw)

        for img in imgs:
            self.urls.append(img.url)
            self.filenames[img.url] = img.filename

        self.title = self.name


class Image:
    def __init__(self, id_, url):
        self.url = url
        ext = os.path.splitext(url)[1]
        self.filename = '{}{}'.format(id_, ext)


def setPage(url, page):
    # Always use HTTPS
    url = url.replace('http://', 'https://')

    # Change the page
    if 'pid=' in url:
        url = re.sub('pid=[0-9]*', 'pid={}'.format(page), url)
    else:
        url += '&pid={}'.format(page)

    return url


def get_imgs(url, title=None, cw=None):
    url = clean_url(url)
    if 's=view' in url and 'page=favorites' not in url:
        raise NotImplementedError('Not Implemented')

    if 'page=dapi' not in url.lower():
        tags = get_tags(url)
        tags = quote(tags, safe='/')
        tags = tags.replace('%20', '+')
        url = "https://rule34.xxx/index.php?page=dapi&s=post&q=index&tags={}&pid={}&limit={}".format(tags, 0, LIMIT)

    print_ = get_print(cw)

    # Range
    max_pid = get_max_range(cw)

    imgs = []
    ids = set()
    for p in range(500): #1017
        check_alive(cw)
        url = setPage(url, p)
        print_(url)
        html = try_n(4, sleep=30)(downloader.read_html)(url) #3340

        soup = Soup(html)
        posts = soup.findAll('post')
        if not posts:
            break
        for post in posts:
            id_ = post.attrs['id']
            if id_ in ids:
                print('duplicate:', id_)
                continue
            ids.add(id_)
            url_img = post.attrs['file_url']
            img = Image(id_, url_img)
            imgs.append(img)
        if len(imgs) >= max_pid:
            break

        if cw is not None:
            cw.setTitle('{}  {} - {}'.format(tr_('읽는 중...'), title, len(imgs)))
    return imgs



File: src/extractor/weibo_downloader.py

#coding:utf8
import downloader
import ree as re
from utils import Downloader, Session, get_print, clean_title, Soup, fix_protocol, domain, get_max_range, get_ext, File, check_alive, limits
from translator import tr_
import clf2
import errors
import utils
import dateutil.parser


def suitable(url):
    if domain(url.lower(), 2) not in ['weibo.com', 'weibo.cn']:
        return False
    if '/tv/' in url.lower():
        return False
    return True


class LoginRequired(errors.LoginRequired):
    def __init__(self, *args):
        super().__init__(*args, method='browser', url='https://weibo.com/login.php', w=1180)



class Downloader_weibo(Downloader):
    type = 'weibo'
    URLS = [suitable]
    MAX_PARALLEL = 2 #6739
    ACCEPT_COOKIES = [r'(.*\.)?(weibo\.com|sina\.com\.cn|weibo\.cn)']

    def init(self):
        self.session = Session()

    @classmethod
    def fix_url(cls, url):
        url = url.replace('weibo.cn', 'weibo.com').split('?')[0]
        if 'weibo.com/p/' in url:
            id = re.find(r'weibo.com/p/([^/]+)', url, err='no id')
            url = f'https://weibo.com/p/{id}'
        elif 'weibo.com/u/' in url:
            id = re.find(r'weibo.com/u/([^/]+)', url, err='no id')
            url = f'https://weibo.com/u/{id}'
        elif 'weibo.com/' in url:
            id = re.find(r'weibo.com/([^/]+)', url, err='no id')
            url = f'https://weibo.com/{id}'
        else:
            id = url
            url = f'https://weibo.com/u/{id}'
        return fix_protocol(url)

    def read(self):
        checkLogin(self.session)

        uid, oid, name = get_id(self.url, self.cw)
        title = clean_title(f'{name} (weibo_{uid})')

        self.urls += get_imgs(uid, title, self.session, cw=self.cw)

        self.title = title


def checkLogin(session):
    c = session.cookies._cookies.get('.weibo.com', {}).get('/',{}).get('SUBP')
    if not c or c.is_expired():
        raise LoginRequired()


class Album:

    def __init__(self, id, type):
        self.id = id
        self.type = type


@limits(1)
def wait():
    pass


class Image(File):

    type = 'weibo'
    format = '[date] id_ppage'


def _get_page_id(html):
    return re.find(r"CONFIG\['page_id'\]='([0-9]+)'", html) or re.find(r'/u/page/follow/([0-9]+)', html)


def get_id(url, cw=None):
    for try_ in range(2):
        try:
            res = clf2.solve(url, cw=cw, f=_get_page_id)
            html = res['html']
            soup = Soup(html)
            if soup.find('div', class_='gn_login') or soup.find('a', class_=lambda c: c and c.startswith('LoginBtn')):
                raise LoginRequired()
            oid = _get_page_id(html)
            if not oid:
                raise Exception('no page_id')
            uids = re.findall(r'uid=([0-9]+)', html)
            uid = max(set(uids), key=uids.count)
            name = re.find(r"CONFIG\['onick'\]='(.+?)'", html) or soup.find('div', class_=lambda c:c and c.startswith('ProfileHeader_name')).text.strip()
            if not name:
                raise Exception('no name')
            break
        except errors.LoginRequired as e:
            raise e
        except Exception as e:
            e_ = e
            print(e)
    else:
        raise e_
    return uid, oid, name


def extract_video(d):
    return d.get('stream_url_hd') or d['stream_url']


def get_imgs(uid, title, session, cw=None): #6739
    print_ = get_print(cw)
    print_(f'uid: {uid}')

    olds = utils.process_olds(Image, title, r'([0-9]+)_p', cw)
    mids = olds['ids']
    imgs_old = olds['imgs']

    referer = f'https://weibo.com/u/{uid}?tabtype=album'
    imgs = []
    sinceid = None

    while check_alive(cw):
        if sinceid:
            url_api = f'https://weibo.com/ajax/profile/getImageWall?uid={uid}&sinceid={sinceid}'
        else:
            url_api = f'https://weibo.com/ajax/profile/getImageWall?uid={uid}&sinceid=0&has_album=true'
        wait()
        d = downloader.read_json(url_api, referer, session=session)
        sinceid = d['data']['since_id']
        for item in d['data']['list']:
            mid = int(item['mid'])
            if mid in mids:
                #print_(f'dup: {mid}')
                continue
            mids.add(mid)
            url_api = f'https://weibo.com/ajax/statuses/show?id={mid}'
            wait()
            d = downloader.read_json(url_api, referer, session=session)
            if d.get('ok') != 1:
                print_(f'skip: {mid}')
                continue
            date = dateutil.parser.parse(d['created_at'])
            structs = [d] + (d.get('url_struct') or [])
            for struct in structs:
                media_info = struct.get('mix_media_info', {}).get('items') or (struct.get('pic_infos').values() if 'pic_infos' in struct else None) #6739
                if media_info:
                    break
            else:
                print_(f'no media: {mid}') #6739
                continue
            for p, item in enumerate(media_info):
                if data := item.get('data'):
                    type = item.get('type')
                    if type == 'video':
                        img = extract_video(data['media_info'])
                    elif type == 'pic':
                        img = data['largest']['url']
                    else:
                        raise Exception(f'media type: {type}')
                else:
                    img = item['largest']['url']
                ext = get_ext(img)
                d = {
                    'date': date,
                    'id': mid,
                    'page': p,
                    }
                filename = utils.format('weibo', d, ext)
                img = Image({'referer': referer, 'url': img, 'name': filename})
                imgs.append(img)

            cw.setTitle(f'{tr_("읽는 중...")}  {title} - {len(imgs)}')

        if not sinceid:
            break

        if len(imgs) >= get_max_range(cw):
            break

    return imgs + imgs_old



File: src/extractor/youku_downloader.py

import downloader
import ytdl
from m3u8_tools import M3u8_stream
from utils import LazyUrl, Downloader, format_filename
from io import BytesIO



class Downloader_youku(Downloader):
    type = 'youku'
    single = True
    URLS = ['v.youku.com']
    ACCEPT_COOKIES = [r'(.*\.)?youku\.com']

    def read(self):
        video = Video(self.url, cw=self.cw)
        video.url()# get thumb

        self.urls.append(video.url)
        self.setIcon(video.thumb)

        self.title = video.title


class Video:
    _url = None

    def __init__(self, url, cw=None):
        self.url = LazyUrl(url, self.get, self)
        self.cw = cw

    def get(self, url):
        if self._url:
            return self._url

        ydl = ytdl.YoutubeDL(cw=self.cw)
        info = ydl.extract_info(url)

        # get best video
        fs = info['formats']
        fs = sorted(fs, key=lambda x: int(x['width']), reverse=True)
        f = fs[0]
        url_video = f['url']

        # thumb
        self.thumb_url = info['thumbnails'][0]['url']
        self.thumb = BytesIO()
        downloader.download(self.thumb_url, buffer=self.thumb)

        # m3u8
        print(f['protocol'])
        if 'm3u8' in f['protocol']:
            url_video = M3u8_stream(url_video, referer=url)

        # title & filename
        self.title = info['title']
        self.filename = format_filename(self.title, info['id'], '.mp4')

        self._url = url_video

        return self._url



File: src/extractor/hentaicosplay_downloader.py

#coding: utf8
import downloader
from utils import Downloader, Session, Soup, LazyUrl, urljoin, get_ext, clean_title, try_n, limits
import utils
import ree as re
from translator import tr_
import clf2
from m3u8_tools import M3u8_stream
from timee import sleep
import os



class Image:
    def __init__(self, url, referer, p, session):
        self._url = url
        self._referer = referer
        self._p = p
        self.url = LazyUrl(url, self.get, self)
        self.session = session

    @try_n(3, 5)
    @limits(1)
    def get(self, _=None):
        soup = downloader.read_soup(self._url, self._referer, session=self.session)
        div = soup.find('div', id='display_image_detail') or soup.find('ul', id='detail_list')
        parent = div.find('img').parent
        while not parent.get('href'):
            parent = parent.parent
        url = urljoin(self._url, parent['href'])
        ext = get_ext(url)
        self.filename = '{:04}{}'.format(self._p, ext)
        return url, self._url


class Video:

    def __init__(self, src, referer, title, session):
        ext = get_ext(src)
        if ext == '.m3u8':
            _src = src
            src = M3u8_stream(_src, referer=referer, session=session)
            ext = '.mp4'
        self.url = LazyUrl(referer, lambda _: src, self)
        self.filename = '{}{}'.format(clean_title(title), ext)



class Downloader_hentaicosplay(Downloader):
    type = 'hentaicosplay'
    URLS = ['hentai-cosplays.com', 'porn-images-xxx.com', 'hentai-img.com']
    icon = None
    display_name = 'Hentai Cosplay'
    MAX_PARALLEL = 1 # must be 1
    MAX_CORE = 4
    ACCEPT_COOKIES = [rf'(.*\.)?{domain}' for domain in URLS]

    @classmethod
    def fix_url(cls, url):
        url = re.sub(r'/page/[0-9]+', '', url)
        url = re.sub(r'/attachment/[0-9]+', '', url)
        url = re.sub(r'([a-zA-Z]+\.)hentai-cosplays\.com', 'hentai-cosplays.com', url)
        url = re.sub(r'.com/story/', '.com/image/', url)
        return url

    def init(self):
        self.session = Session()

    @try_n(2)
    def read(self):
        #4961
        ua = downloader.random_ua()
        self.print_(f'read start ua: {ua}')
        downloader.REPLACE_UA[r'hentai-cosplays\.com'] = ua
        downloader.REPLACE_UA[r'porn-images-xxx\.com'] = ua

        if '/video/' in self.url:
            res = clf2.solve(self.url, session=self.session, cw=self.cw)
            soup = Soup(res['html'])
            title = (soup.find('h1', id='post_title') or soup.find('div', id='page').find('h2')).text.strip()
            self.title = title
            view = soup.find('div', id='post') or soup.find('div', class_='video-container')
            video = view.find('video')
            src = video.find('source')['src']
            src = urljoin(self.url, src)
            video = Video(src, self.url, title, self.session)
            self.urls.append(video.url)
            self.single = True
            return

        if '/image/' not in self.url:
            raise NotImplementedError('Not a post')

        res = clf2.solve(self.url, session=self.session, cw=self.cw)
        soup = Soup(res['html'])
        title = (soup.find('h2') or soup.find('h3')).text
        paginator = soup.find('div', id='paginator') or soup.find('div', class_='paginator_area')
        pages = [self.url]
        for a in paginator.findAll('a'):
            href = a.get('href')
            if not href:
                continue
            href = urljoin(self.url, href)
            if href not in pages:
                pages.append(href)
        self.print_(f'pages: {len(pages)}')

        imgs = []
        for i, page in enumerate(pages):
            sleep(2, self.cw)
            if page == self.url:
                soup_page =  soup
            else:
                soup_page = try_n(3, 5)(downloader.read_soup)(page, session=self.session)
            view = soup_page.find('div', id='post') or soup_page.find('ul', id='detail_list')
            for img in view.findAll('img'):
                href = img.parent.get('href') or img.parent.parent.get('href')
                if not href:
                    continue
                href = urljoin(page, href)
                img = Image(href, page, len(imgs), self.session)
                imgs.append(img)
            self.print_(f'imgs: {len(imgs)}')
            self.cw.setTitle('{} {} ({} / {})'.format(tr_('읽는 중...'), title, i+1, len(pages)))

        names = {}
        dirname = utils.dir(self.type, clean_title(title), self.cw)
        try:
            files = os.listdir(dirname)
        except:
            files = []
        for file in files:
            name, ext = os.path.splitext(file)
            names[name] = ext

        for p, img in enumerate(imgs):
            name = '{:04}'.format(p)
            ext = names.get(name)
            if ext:
                self.urls.append(os.path.join(dirname, '{}{}'.format(name, ext)))
            else:
                self.urls.append(img.url)

        self.title = clean_title(title)



File: src/extractor/tokyomotion_downloader.py

#coding:utf8
import downloader
from utils import Soup, Downloader, LazyUrl, clean_title, format_filename
from io import BytesIO
import ree as re
import os


class Downloader_tokyomotion(Downloader):
    type = 'tokyomotion'
    URLS = ['tokyomotion.net']
    single = True
    _type = None
    display_name = 'TOKYO Motion'
    ACCEPT_COOKIES = [r'(.*\.)?tokyomotion\.net']

    def init(self):
        html = downloader.read_html(self.url)
        self.soup = Soup(html)
        if '/album/' in self.url:
            self._type = 'album'
        else:
            self._type = 'video'

    @property
    def name(self):
        title = get_title(self.soup)
        return clean_title(title)

    def read(self):
        if self._type == 'video':
            video = get_video(self.url, self.soup)
            self.urls.append(video.url)
            self.setIcon(video.thumb)
        elif self._type == 'album':
            imgs = get_imgs(self.url)
            for img in imgs:
                self.urls.append(img.url)
            self.single = False
        else:
            raise NotImplementedError('Unknown type: {}'.format(self._type))

        self.title = self.name


class Video:
    def __init__(self, url, url_thumb, referer, filename):
        self.url = LazyUrl(referer, lambda x: url, self)
        self.url_thumb = url_thumb
        self.thumb = BytesIO()
        downloader.download(url_thumb, referer=referer, buffer=self.thumb)
        self.filename = filename


def get_title(soup):
    video = soup.find('video', id='vjsplayer')
    if video:
        title = soup.find('h3').text.strip()
    else:
        title = soup.find('title').text.split(' Album - ')[0].strip()
    return title


def get_video(url, soup=None):
    if soup is None:
        html = downloader.read_html(url)
        soup = Soup(html)

    video = soup.find('video', id='vjsplayer').find('source').attrs['src']
    url_thumb = soup.find('video', id='vjsplayer').attrs['poster']
    title = get_title(soup)
    filename = format_filename(title, '', '.mp4')
    video = Video(video, url_thumb, url, filename)
    return video


class Image:
    def __init__(self, url, referer):
        self.url = LazyUrl(referer, lambda x: url, self)
        self.filename = os.path.basename(url.split('?')[0])


def get_imgs(url):
    id = re.find('album/.*?([0-9]+)', url)
    print('id:', id)
    url = 'https://www.tokyomotion.net/album/slideshow/{}'.format(id)

    html = downloader.read_html(url)
    soup = Soup(html)

    imgs = []
    for a in soup.findAll('a', {'data-lightbox': 'slideshow-{}'.format(id)}):
        img = a.find('img').attrs['src']
        img = img.replace('/tmb/', '/')
        img = Image(img, url)
        imgs.append(img)

    return imgs



File: src/extractor/syosetu_downloader.py

#coding:utf8
import downloader
import utils
from utils import urljoin, try_n, Downloader, clean_title, Session, File, check_alive, get_max_range
import ree as re
from io import BytesIO
import os
from translator import tr_
from timee import sleep


class Text(File):
    type = 'syosetu'
    format = 'title'

    def __init__(self, info):
        title = info['subtitle']
        if not info['single']:
            p = int(re.findall('/([0-9]+)', info['referer'])[-1])
            title = clean_title(f'[{p:04}] {title}')
        info['title_all'] = title
        d = {
            'title': info['title_all'],
            }
        info['name'] = utils.format(self.type, d, '.txt')
        super().__init__(info)

    def get(self):
        text = get_text(self['referer'], self['title_all'], self['update'], self.session)
        f = BytesIO()
        f.write(text.encode('utf8'))
        f.seek(0)
        return {'url': f}



def get_id(url):
    return re.find(r'.com/([^/]+)', url) or url



class Downloader_syosetu(Downloader):
    type = 'syosetu'
    URLS = ['syosetu.com']
    MAX_CORE = 2
    detect_removed = False
    display_name = '小説家になろう'
    ACCEPT_COOKIES = [r'(.*\.)?syosetu\.com']
    atts = ['_title_', 'novel_ex']

    @classmethod
    def fix_url(cls, url):
        return f'https://ncode.syosetu.com/{get_id(url)}/'

    def read(self):
        for try_ in range(8):
            self.print_('get_session')
            try:
                self.session = get_session()
                self.purge_cookies()
                soup = downloader.read_soup(self.url, session=self.session)
                get_title_artist(soup)
                break
            except Exception as e:
                print(e)

        else:
            raise

        title, self.artist = get_title_artist(soup)
        self._title_ = title
        ncode = re.find(r'syosetu.com/([^/]+)', self.url, err='no ncode') #3938
        title_dir = clean_title(f'[{self.artist}] {title} ({ncode})')
        ex = soup.find('div', id='novel_ex')
        self.novel_ex = utils.get_text(ex, '') if ex else None
        texts = []

        # Range
        max_pid = get_max_range(self.cw)

        while check_alive(self.cw):
            subtitles = soup.findAll('dd', class_='subtitle')
            if subtitles:
                for subtitle in subtitles:
                    update = subtitle.parent.find('dt', class_='long_update')
                    update2 = None
                    if update:
                        for span in update.findAll('span'):
                            update2 = span.attrs['title']
                            span.decompose()

                        update = update.text.strip()
                    if update2:
                        update += f'  ({update2})'
                    a = subtitle.find('a')
                    subtitle = a.text.strip()
                    href = urljoin(self.url, a.attrs['href'])
                    if not re.search(f'ncode.syosetu.com/{get_id(self.url)}/[0-9]+', href):
                        self.print_(f'skip: {href}')
                        continue
                    text = Text({'referer': href, 'subtitle': subtitle, 'update': update, 'single': False})
                    texts.append(text)
            else:
                self.single = True
                text = Text({'referer': self.url, 'subtitle': title_dir, 'update': None, 'single': True})
                texts.append(text)
            if len(texts) >= max_pid:
                break
            if pager_next := soup.find('a', class_='novelview_pager-next'): #6830
                sleep(1)
                url_next = urljoin(self.url, pager_next['href'])
                self.print_(f'url_next: {url_next}')
                soup = downloader.read_soup(url_next, self.url, session=self.session)
            else:
                break
        self.print_(f'single: {self.single}')
        self.urls += texts

        self.title = title_dir

    def post_processing(self):
        if self.single:
            return
        names = self.cw.names
        filename = os.path.join(self.dir, f'[merged] {self.title}.txt')
        try:
            with utils.open(filename, 'wb') as f:
                f.write(f'    {self._title_}\n\n    \u4f5c\u8005\uff1a{self.artist}\n\n\n'.encode('utf8'))
                if self.novel_ex:
                    f.write(self.novel_ex.encode('utf8'))
                for i, file in enumerate(names):
                    self.cw.pbar.setFormat(f'[%v/%m]  {tr_("병합...")} [{i}/{len(names)}]')
                    with open(file, 'rb') as f_:
                        text = f_.read()
                    f.write(b'\n\n\n\n')
                    f.write(text)
        finally:
            self.cw.pbar.setFormat("[%v/%m]")


def get_title_artist(soup):
    artist = soup.find('div', class_='novel_writername').text.replace('\u4f5c\u8005', '').replace('\uff1a', '').replace(':', '').replace('\u3000', ' ').strip()
    rem = len(artist.encode('utf8', 'ignore')) + len('[merged] [] .txt') + len(' (n8273ds)')
    return clean_title(soup.find('p', class_='novel_title').text.strip(), n=-rem), clean_title(artist)


@try_n(22, sleep=30)
def get_text(url, subtitle, update, session):
    soup = downloader.read_soup(url, session=session)
    if update:
        update = '        ' + update
    else:
        update = ''

    story = utils.get_text(soup.find('div', id='novel_honbun'), '')

    p = soup.find('div', id='novel_p')
    p = '' if p is None else utils.get_text(p, '')
    if p:
        story = f'{p}\n\n════════════════════════════════\n\n{story}'

    #2888
    a = soup.find('div', id='novel_a')
    a = '' if a is None else utils.get_text(a, '')
    if a:
        story = f'{story}\n\n════════════════════════════════\n\n{a}'

    text = f'''────────────────────────────────

  ◆  {subtitle}{update}

────────────────────────────────


{story}'''
    return text


def get_session():
    session = Session()
    session.cookies.set(name='over18', value='yes', path='/', domain='.syosetu.com')
    return session



File: src/extractor/bdsmlr_downloader.py

#coding:utf8
import downloader
from utils import Session, Soup, LazyUrl, Downloader, get_max_range, try_n, get_print, clean_title, check_alive
from datetime import datetime
import ree as re
import os
from translator import tr_
from error_printer import print_error
import clf2
import errors



class Downloader_bdsmlr(Downloader):
    type = 'bdsmlr'
    URLS = ['bdsmlr.com']
    display_name = 'BDSMlr'
    ACCEPT_COOKIES = [r'(.*\.)?bdsmlr\.com']

    def init(self):
        if 'bdsmlr.com/post/' in self.url:
            raise errors.Invalid(tr_('개별 다운로드는 지원하지 않습니다: {}').format(self.url))

        self.url = 'https://{}.bdsmlr.com'.format(self.id_)
        self.session = Session()
        clf2.solve(self.url, session=self.session,  cw=self.cw)

    @property
    def id_(self):
        url = self.url
        if 'bdsmlr.com' in url:
            if 'www.bdsmlr.com' in url:
                raise Exception('www.bdsmlr.com')
            gal_num = url.split('.bdsmlr.com')[0].split('/')[(-1)]
        else:
            gal_num = url
        return gal_num

    def read(self):
        info = get_imgs(self.id_, session=self.session, cw=self.cw)

        for post in info['posts']:
            self.urls.append(post.url)

        self.title = '{} (bdsmlr_{})'.format(clean_title(info['username']), self.id_)


class Post:
    def __init__(self, url, referer, id, p):
        self.id = id
        self.url = LazyUrl(referer, lambda x: url, self)
        ext = os.path.splitext(url)[1]
        self.filename = '{}_p{}{}'.format(id, p, ext)


def foo(url, soup, info, reblog=False):
    #print('foo', info['c'], len(info['ids']))
    for post in soup.findAll('div', class_='wrap-post'):
        try:
            id = int(re.find('[0-9]+', post.attrs['class'][1]))
        except Exception as e:
            print(print_error(e))
            continue
        if id in info['ids']:
            continue
        info['ids'].add(id)
        info['last'] = id
        if not reblog and post.find('div', class_='ogname'):
            continue
        for p, mag in enumerate(post.findAll(['a', 'div'], class_='magnify')):
            post = Post(mag.attrs['href'], url, id, p)
            info['posts'].append(post)
    info['c'] += 20 if info['c'] else 5


@try_n(2)
def get_imgs(user_id, session, cw=None):
    print_ = get_print(cw)
    url = 'https://{}.bdsmlr.com/'.format(user_id)
    info = {'c': 0, 'posts': [], 'ids': set()}

    html = downloader.read_html(url, session=session)
    soup = Soup(html)

    sorry = soup.find('div', class_='sorry')
    if sorry:
        raise Exception(sorry.text.strip())

    username = soup.find('title').text.strip()###
    print('username:', username)
    info['username'] = username

    token = soup.find('meta', {'name': 'csrf-token'}).attrs['content']
    print_('token: {}'.format(token))

    max_pid = get_max_range(cw)

    n = len(info['ids'])
    for p in range(1000):
        check_alive(cw)
        if p == 0:
            url_api = 'https://{}.bdsmlr.com/loadfirst'.format(user_id)
        else:
            url_api = 'https://{}.bdsmlr.com/infinitepb2/{}'.format(user_id, user_id)
        data = {
            'scroll': str(info['c']),
            'timenow': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
            }
        if 'last' in info:
            data['last'] = str(info['last'])
        print_('n:{}, scroll:{}, last:{}'.format(len(info['posts']), data['scroll'], data.get('last')))
        headers = {
            'Referer': url,
            'X-CSRF-TOKEN': token,
            }
        _e = None
        for try_ in range(4):
            try:
                r = session.post(url_api, data=data, headers=headers)
                if p == 0:
                    r.raise_for_status()
                break
            except Exception as e:
                _e = e
                print(e)
        else:
            if _e is not None:
                raise _e
        soup = Soup(r.text)
        foo(url, soup, info)
        if len(info['ids']) == n:
            print('same; break')
            break
        n = len(info['ids'])

        s = '{}  {} (tumblr_{}) - {}'.format(tr_('읽는 중...'), username, user_id, len(info['posts']))
        if cw is not None:
            cw.setTitle(s)
        else:
            print(s)

        if len(info['posts']) > max_pid:
            break

    return info



File: src/extractor/webtoon_downloader.py

import downloader
from utils import Soup, Session, LazyUrl, clean_title, get_ext, get_imgs_already, urljoin, try_n, Downloader, check_alive
import page_selector
from translator import tr_
import ree as re
import clf2



class Downloader_webtoon(Downloader):
    type = 'webtoon'
    URLS = ['webtoon.com', 'webtoons.com']
    MAX_CORE = 8
    MAX_SPEED = 4.0
    display_name = 'WEBTOON'
    ACCEPT_COOKIES = [r'(.*\.)?webtoons?\.com']

    def init(self):
        self.session = Session()
        clf2.solve(self.url, session=self.session)
        self.url = get_main(self.url, self.session)
        self.soup = downloader.read_soup(self.url, session=self.session)

    @classmethod
    def fix_url(cls, url):
        return url.replace('webtoon.com', 'webtoons.com')

    def read(self):
        title = clean_title(self.soup.find('h1').text.strip())
        self.title = tr_('읽는 중... {}').format(title)
        imgs = get_imgs_all(self.url, self.session, title, cw=self.cw)
        for img in imgs:
            if isinstance(img, Image):
                self.urls.append(img.url)
            else:
                self.urls.append(img)

        self.title = title


class Page:

    def __init__(self, url, title):
        self.url = url
        self.title = title


class Image:

    def __init__(self, url, session, page, p):
        ext = get_ext(url) or downloader.get_ext(url, referer=page.url, session=session)
        self.filename = '{}/{:04}{}'.format(clean_title(page.title), p, ext)

        self.url = LazyUrl(page.url, lambda _: url, self)


@try_n(2)
def get_imgs(page, session):
    html = downloader.read_html(page.url, session=session)
    if 'window.__motiontoonViewerState__' in html:
        raise NotImplementedError('motiontoon')
    soup = Soup(html)
    view = soup.find('div', class_='viewer_img')
    imgs = []
    for img in view.findAll('img'):
        src = img.get('data-url') or img['src']
        img = Image(urljoin(page.url, src), session, page, len(imgs))
        imgs.append(img)
    return imgs


def get_main(url, session):
    if 'episode_no=' in url:
        soup = downloader.read_soup(url, session=session)
        url = urljoin(url, soup.find('div', class_='subj_info').find('a')['href'])
    return url


def set_page(url, p):
    if '&page=' not in url:
        url = url + '&page={}'.format(p)
    else:
        url = re.sub('&page=[0-9]+', '&page={}'.format(p), url)
    if p == 1:
        url = url.replace('&page=1', '')
    return url


def get_pages(url, session=None):
    pages = []
    urls = set()
    for p in range(1, 101):
        url_page = set_page(url, p)
        print(url_page)
        for try_ in range(4):
            try:
                soup = downloader.read_soup(url_page, session=session)
                view = soup.find('ul', id='_listUl')
                if view is None:
                    raise Exception('no view')
                break
            except Exception as e:
                e_ = e
                print(e)
        else:
            raise e_
        pages_new = []
        for li in view.findAll('li', recursive=False):
            href = urljoin(url, li.find('a')['href'])
            title = li.find('span', class_='subj').text.strip()
            if href in urls:
                continue
            urls.add(href)
            no = int(li['data-episode-no'])
            title = '{:04} - {}'.format(no, title)
            page = Page(href, title)
            pages_new.append(page)
        if not pages_new:
            break
        pages += pages_new
    return pages[::-1]


@page_selector.register('webtoon')
@try_n(4)
def f(url):
    url = get_main(url, None)
    return get_pages(url)


def get_imgs_all(url, session, title, cw=None):
    pages = get_pages(url, session)
    pages = page_selector.filter(pages, cw)
    imgs = []
    for p, page in enumerate(pages):
        check_alive(cw)
        imgs_already = get_imgs_already('webtoon', title, page, cw)
        if imgs_already:
            imgs += imgs_already
            continue
        imgs += get_imgs(page, session)
        msg = tr_('읽는 중... {} / {}  ({}/{})').format(title, page.title, p + 1, len(pages))
        if cw is not None:
            cw.setTitle(msg)
        else:
            print(msg)

    return imgs



File: src/extractor/avgle_downloader.py

#coding: utf8
import downloader
from m3u8_tools import M3u8_stream
from utils import Soup, Downloader, LazyUrl, get_print, try_n, check_alive, format_filename, json
from io import BytesIO
import base64
import webbrowser
import errors



class Downloader_avgle(Downloader):
    type = 'avgle'
    single = True
    URLS = ['avgle.com']
    ACCEPT_COOKIES = [r'(.*\.)?avgle\.com']

    def init(self):
        if not self.cw.data_:
            link = 'https://github.com/KurtBestor/Hitomi-Downloader/wiki/Chrome-Extension'
            webbrowser.open(link)
            raise errors.Invalid('No data; See: {}'.format(link))

    def read(self):
        video = get_video(self.url, cw=self.cw)
        self.urls.append(video.url)

        self.setIcon(video.thumb)

        self.title = video.title


@try_n(2)
def get_video(url, cw=None):
    print_ = get_print(cw)

    check_alive(cw)

    data = cw.data_
    version = data['version']
    print_('version: {}'.format(version))
    if version == '0.1':
        raise errors.OutdatedExtension()
    data = data['data']
    if not isinstance(data, bytes):
        data = data.encode('utf8')
    s = base64.b64decode(data).decode('utf8')
    urls = json.loads(s)

    print_('\n'.join(urls[:4]))

    referer_seg = 'auto' if 'referer=force' in urls[0] else None # 1718

    stream = M3u8_stream(url, urls=urls, n_thread=4, referer_seg=referer_seg)

    html = downloader.read_html(url)
    soup = Soup(html)

    url_thumb = soup.find('meta', {'property': 'og:image'}).attrs['content']
    title = soup.find('meta', {'property': 'og:title'}).attrs['content'].strip()

    video = Video(stream, url_thumb, url, title)

    return video


class Video:
    def __init__(self, url, url_thumb, referer, title):
        self.url = LazyUrl(referer, lambda x: url, self)
        self.url_thumb = url_thumb
        self.thumb = BytesIO()
        downloader.download(url_thumb, referer=referer, buffer=self.thumb)
        self.title = title
        self.filename = format_filename(title, '', '.mp4')



File: src/extractor/nhentai_downloader.py

#coding:utf8
import downloader
import ree as re
from utils import urljoin, File, Downloader, try_n, join, get_ext, json
import utils
import clf2


def get_id(url):
    try:
        return int(url)
    except:
        return int(re.find('/g/([0-9]+)', url))


class File_nhentai(File):
    type = 'nhentai'
    format = 'page:04;'


class Downloader_nhentai(Downloader):
    type = 'nhentai'
    URLS = ['nhentai.net']
    MAX_CORE = 16
    display_name = 'nhentai'
    ACCEPT_COOKIES = [r'(.*\.)?nhentai\.net']

    def init(self):
        self.session = clf2.solve(self.url, cw=self.cw)['session'] #4541

    @classmethod
    def fix_url(cls, url):
        return f'https://nhentai.net/g/{get_id(url)}/'

    def read(self):
        info, imgs = get_imgs(get_id(self.url), self.session)

        # 1225
        artist = join(info.artists)
        self.artist = artist if info.artists else None
        group = join(info.groups)
        lang = info.lang or 'N／A'
        series = info.seriess[0] if info.seriess else 'N／A'
        title = self.format_title(info.type, info.id, info.title, artist, group, series, lang)

        self.urls += imgs

        self.title = title


class Info:
    def __init__(self, host, id, id_media, title, p, artists, groups, seriess, lang, type, formats):
        self.host = host
        self.id = id
        self.id_media = id_media
        self.title = title
        self.p = p
        self.artists = artists
        self.groups = groups
        self.seriess = seriess
        self.lang = lang
        self.type = type
        self.formats = formats


@try_n(4)
def get_info(id, session):
    url = f'https://nhentai.net/g/{id}/1/'
    referer = f'https://nhentai.net/g/{id}/'
    html = downloader.read_html(url, referer, session=session)

    data = html.split('JSON.parse(')[1].split(');')[0]
    gal = json.loads(json.loads(data))
    host = 'https://i.nhentai.net'#re.find('''media_url: *['"]([^'"]+)''', html, err='no host')

    id = int(gal['id'])
    id_media = int(gal['media_id'])
    title = gal['title']['english']
    p = len(gal['images']['pages'])
    artists = []
    groups = []
    seriess = []
    for tag in gal['tags']:
        type = tag['type']
        if type == 'artist':
            artists.append(tag['name'])
        elif type == 'group':
            groups.append(tag['name'])
        elif type == 'parody' and tag['name'] != 'original':
            seriess.append(tag['name'])
        elif type == 'language':
            lang = tag['name']
        elif type == 'category':
            type_ = tag['name']
    formats = []
    for img in gal['images']['pages']:
        type = img['t']
        format = {'j':'jpg', 'p':'png', 'g':'gif'}[type]
        formats.append(format)
    info = Info(host, id, id_media, title, p, artists, groups, seriess, lang, type_, formats)
    return info


def get_imgs(id, session):
    info = get_info(id, session)

    imgs = []
    for p in range(1, info.p+1):
        name = f'/galleries/{info.id_media}/{p}.{info.formats[p-1]}'
        url_page = f'https://nhentai.net/g/{id}/{p}/'
        url_img = urljoin(info.host, name)
        ext = get_ext(url_img)
        d = {
            'page': p,
            }
        img = File_nhentai({'url': url_img, 'referer': url_page, 'name': utils.format('nhentai', d, ext)})
        imgs.append(img)

    return info, imgs



File: src/extractor/nico_downloader.py

#coding:utf8
import downloader
from io import BytesIO
import ree as re
from utils import Downloader, get_print, format_filename, try_n, LazyUrl, get_abr, Session, get_resolution, print_error, urljoin
import utils
import ffmpeg
import os
import ytdl
import threading
import errors
import websockets # for nama
from m3u8_tools import M3u8_stream


def get_id(url):
    if '/watch/' in url:
        return re.find('/watch/([a-zA-Z0-9]+)', url)


class LoginRequired(errors.LoginRequired):
    def __init__(self, *args):
        super().__init__(*args, method='browser', url='https://account.nicovideo.jp/login')


class Video:
    def __init__(self, session, info, format, cw, hb=None, d=None, live=False, ydl=None):
        self.session = session
        self.info = info
        self.title = info.get('fulltitle') or info['title']
        self.ext = info['ext']
        self.id = info['id']
        self.format = format
        self.username = info['uploader']
        self.url = LazyUrl(f'https://www.nicovideo.jp/watch/{self.id}', self.get, self, pp=self.pp)
        self.cw = cw
        self.hb = hb
        self.d = d
        self.live = live
        self.ydl = ydl

        self.filename = format_filename(self.title, self.id, self.ext, live=live, artist=self.username)

        self.url_thumb = info['thumbnail']
        print('thumb:', self.url_thumb)
        self.thumb = BytesIO()
        downloader.download(self.url_thumb, buffer=self.thumb)

    def get(self, _):
        print_ = get_print(self.cw)
        hb = self.hb
        if hb:
            heartbeat_info_dict = hb['info']
            heartbeat_url = heartbeat_info_dict['url']
            heartbeat_data = heartbeat_info_dict['data'].encode()
            heartbeat_interval = heartbeat_info_dict.get('interval', 30)

            request = ytdl.get_ytdl().utils.sanitized_Request(heartbeat_url, heartbeat_data)

            def heartbeat():
                if self.d.status == 'stop':
                    print_('Heartbeat end')
                    return
                try:
                    hb['ydl'].urlopen(request).read()
                except Exception as e:
                    e_msg = print_error(e)
                    print_(f'Heartbeat failed:\n{e_msg}')

                self.timer = threading.Timer(heartbeat_interval, heartbeat)
                self.timer.start()

            heartbeat_info_dict['ping']()
            print_('Heartbeat with %d second interval ...' % heartbeat_interval)
            heartbeat()
        url = self.info['url']
        if self.live:
            url = ytdl.Downloader(self.ydl, self.info, self.info['format_'], live=True, cw=self.cw)
        return url

    def pp(self, filename):
        if self.format == 'mp4':
            return
        name, ext_old = os.path.splitext(filename)
        filename_new = f'{name}.mp3'
        ffmpeg.convert(filename, filename_new, f'-shortest -preset ultrafast -b:a {get_abr()}k', cw=self.cw)

        if utils.ui_setting.albumArt.isChecked():
            self.thumb.seek(0)#
            ffmpeg.add_cover(filename_new, self.thumb, {'artist':self.username, 'title':self.title}, cw=self.cw)

        return filename_new

    def __repr__(self):
        return f'Video({self.id})'


def suitable(url):
    if 'nicovideo.jp' not in url.lower():
        return False
    if 'nicovideo.jp/user/' in url.lower():
        return True
    return get_id(url) is not None



class Downloader_nico(Downloader):
    type = 'nico'
    single = True
    URLS = [suitable, 'ch.nicovideo.jp']
    display_name = 'Niconico'
    _format = 'mp4'
    MAX_SPEED = 2.0
    ACCEPT_COOKIES = [r'(.*\.)?nicovideo\.jp']

    @classmethod
    def fix_url(cls, url):
        id_ = get_id(url)
        if not id_:
            return url.split('?')[0].split('#')[0]
        if re.find(r'^https?://', id_):
            return url
        if re.find(r'^https?://', url):
            domain = utils.domain(url)
        else:
            domain = 'www.nicovideo.jp'
        return f'https://{domain}/watch/{id_}'

    def init(self):
        self.session = Session('chrome')
        self.url0 = self.url
        if not get_id(self.url):
            self.url = get_live_from_user(self.url, self.session)

    def read(self):
        if self.cw.format:
            self._format = self.cw.format

        if self._format == 'mp3':
            self.cw.setMusic(True)

        video = get_video(self.session, self.url, self._format, self.cw, self)

        self.urls.append(video.url)
        self.setIcon(video.thumb)

        self.title = os.path.splitext(video.filename)[0].replace('：', ':')
        self.artist = video.username

        if video.live:
            d = {}
            d['url'] = self.url0
            d['title'] = video.username
            d['thumb'] = video.thumb.getvalue()
            utils.update_live(d, self.cw)
        else:
            self.enableSegment()


@try_n(2)
def get_video(session, url, format, cw=None, d=None):
    print_ = get_print(cw)

    live = 'live.nico' in url
    if cw and live:
        cw.live = True#

    options = {
        'noplaylist': True,
        #'extract_flat': True,
        'playlistend': 1,
        }
    ydl = ytdl.YoutubeDL(options, cw=cw)
    try:
        info = ydl.extract_info(url)
    except Exception as e:
        e_ = e
        soup = downloader.read_soup(url, session=session)
        box = soup.find('div', class_='channel-invitation-box')
        if box:
            msg = box.find('p', class_='channel-invitation-box-body-channel-desc-msg1')
            if msg:
                msg = msg.text.strip()
            raise LoginRequired(msg or None)
        else:
            raise e_
    fs = info['formats']
    res = max(get_resolution(), min(f.get('height', 0) for f in fs))
    print_(f'res: {res}')
    fs = [f for f in fs if f.get('height', 0) <= res]
    for f in fs:
        print_(f"{f.get('height')} {f['protocol']} {f['format']} - {f['url']}")
##    if not live:
##        fs = [f for f in fs if f['url'].startswith('niconico_dm')]#
    f = fs[-1]
    print_(f'f_url: {f["url"]}')
    if f['url'].startswith('niconico_dmc:'):
        ie = ytdl.get_extractor(url)
        ie._downloader = ydl
        info_dict, heartbeat_info_dict = ie._get_heartbeat_info(f)
        f = info_dict
        hb = {'info': heartbeat_info_dict, 'ydl': ydl}
    elif f['url'].startswith('niconico_dms:'):
        ie = ytdl.get_extractor(url)
        ie._downloader = ydl
        url_m3u8 = ie._get_dms_manifest_url(info)
        print_(f'url_m3u8: {url_m3u8}')
        f['url'] = url_m3u8
        f['protocol'] = 'm3u8'
        _ = info.copy()
        _['formats'] = [f]
        m = ytdl.Downloader(ydl, _, _, cw=cw)
        f['url'] = m
        hb = None
    elif f['protocol'].startswith('m3u8'):
        m = M3u8_stream(f['url'], referer=url, session=session)
##        session.headers.update(f.get('http_headers', {}))
##        hdr = session.headers.copy()
##        m = ffmpeg.Stream(f['url'], headers=hdr, cw=cw)
        f['url'] = m
        hb = None
    else:
        hb = None
    session.headers.update(f.get('http_headers', {}))
    info['url'] = f['url']
    info['format_'] = f

    video = Video(session, info, format, cw, hb=hb, d=d, live=live, ydl=ydl)

    return video


import selector
@selector.options('nico')
def options(urls):
    return [
        {'text': 'MP4 (동영상)', 'format': 'mp4'},
        {'text': 'MP3 (음원)', 'format': 'mp3'},
        ]


def get_live_from_user(url, session):
    if 'ch.nicovideo.jp' in url:
        cid = re.find(r'ch\.nicovideo\.jp/([^/?#]+)', url)
        url = f'https://ch.nicovideo.jp/{cid}/live'
        soup = downloader.read_soup(url, session=session)
        if live_now := soup.find(id='live_now'):
            return urljoin(url, live_now.find('a')['href'])
        raise Exception('no live')
    elif 'nicovideo.jp/user/' in url:
        cid = re.find(r'nicovideo\.jp/user/([^/?#]+)', url)
        d = downloader.read_json(f'https://live.nicovideo.jp/front/api/v1/user-broadcast-history?providerId={cid}&providerType=user&isIncludeNonPublic=false&offset=0&limit=100&withTotalCount=true', session=session)
        for pg in d['data']['programsList']:
            if pg['program']['schedule']['status'] == 'ON_AIR':
                id_ = pg['id']['value']
                return f'https://live.nicovideo.jp/watch/{id_}'
        raise Exception('no live')
    else:
        raise NotImplementedError(url)


class Live_nico(utils.Live):
    type = 'nico'

    @classmethod
    def is_live(cls, url):
        if 'nicovideo.jp/user/' in url.lower():
            return True
        if 'ch.nicovideo.jp' in url.lower():
            return True

    @classmethod
    def fix_url(cls, url):
        if 'nicovideo.jp/user/' in url.lower():
            return '/'.join(url.split('/')[:5]).split('?')[0].split('#')[0]
        return '/'.join(url.split('/')[:4]).split('?')[0].split('#')[0]

    @classmethod
    def check_live(cls, url, info=None):
        try:
            session = Session('chrome')
            get_live_from_user(url, session)
            return True
        except Exception as e:
            print(e)
            return False



<p align="center">
  <img src="imgs/card_crop.png" width="50%"/>
  <br>
</p>

[![GitHub release](https://img.shields.io/github/release/KurtBestor/Hitomi-Downloader.svg?logo=github)](https://github.com/KurtBestor/Hitomi-Downloader/releases/latest)
[![GitHub downloads](https://img.shields.io/github/downloads/KurtBestor/Hitomi-Downloader/latest/total.svg?logo=github)](https://github.com/KurtBestor/Hitomi-Downloader/releases/latest)
[![GitHub downloads](https://img.shields.io/github/downloads/KurtBestor/Hitomi-Downloader/total.svg?logo=github)](https://github.com/KurtBestor/Hitomi-Downloader/releases)

## Links
- [Download](https://github.com/KurtBestor/Hitomi-Downloader/releases/latest)
- [Issues](https://github.com/KurtBestor/Hitomi-Downloader/issues)
- [Scripts & Plugins](https://github.com/KurtBestor/Hitomi-Downloader/wiki/Scripts-&-Plugins)
- [Chrome Extension](https://github.com/KurtBestor/Hitomi-Downloader/wiki/Chrome-Extension)

## Demo
<img src="imgs/how_to_download.gif">

## Features
- 🍰 Simple and clear user interface
- 🚀 Download acceleration
- 💻 Supports 24 threads in a single task
- 🚥 Supports speed limit
- 📜 Supports user scripts
- 🧲 Supports BitTorrent & Magnet
- 🎞️ Supports M3U8 & MPD format videos
- 🌙 Dark mode
- 🧳 Portable
- 📋 Clipboard monitor
- 🗃️ Easy to organize tasks

## Supported Sites
| Site | URL |
| :--: | -- |
| **4chan** | <https://4chan.org> |
| **AfreecaTV** | <https://afreecatv.com> |
| **ArtStation** | <https://artstation.com> |
| **AsmHentai** | <https://asmhentai.com> |
| **Avgle** | <https://avgle.com> |
| **baraag.net** | <https://baraag.net> |
| **半次元** | <https://bcy.net> |
| **BDSMlr** | <https://bdsmlr.com> |
| **bilibili** | <https://bilibili.com> |
| **ComicWalker** | <https://comic-walker.com> |
| **Coub** | <https://coub.com> |
| **Danbooru** | <https://danbooru.donmai.us> |
| **Kakao Webtoon** | <http://webtoon.kakao.com> |
| **DeviantArt** | <https://deviantart.com> |
| **E(x)Hentai Galleries** | <https://e-hentai.org><br><https://exhentai.org> |
| **Facebook** | <https://facebook.com> |
| **FC2 Video** | <https://video.fc2.com> |
| **Flickr** | <https://flickr.com> |
| **Gelbooru** | <https://gelbooru.com> |
| **Hameln** | <https://syosetu.org> |
| **hanime.tv** | <https://hanime.tv> |
| **Hentai Foundry** | <https://hentai-foundry.com> |
| **Hitomi.la** | <https://hitomi.la> |
| **Imgur** | <https://imgur.com> |
| **Instagram** | <https://instagram.com> |
| **Iwara** | <https://iwara.tv><br><https://ecchi.iwara.tv> |
| **Jmana** | <https://jmana.net> |
| **カクヨム** | <https://kakuyomu.jp> |
| **Luscious** | <https://luscious.net> |
| **Mastodon** | <https://mastodon.social> |
| **Misskey** | <https://misskey.io> |
| **MyReadingManga** | <https://myreadingmanga.info> |
| **Naver Blog** | <https://blog.naver.com> |
| **Naver Cafe** | <https://cafe.naver.com> |
| **Naver Post** | <https://post.naver.com> |
| **Naver Webtoon** | <https://comic.naver.com> |
| **Naver TV** | <https://tv.naver.com> |
| **nhentai** | <https://nhentai.net> |
| **nhentai.com** | <https://nhentai.com> |
| **Niconico** | <http://nicovideo.jp> |
| **ニジエ** | <https://nijie.info> |
| **Nozomi.la** | <https://nozomi.la> |
| **Pawoo** | <https://pawoo.net> |
| **Pinterest** | <https://pinterest.com> |
| **Pixiv** | <https://pixiv.net> |
| **pixivコミック** | <https://comic.pixiv.net> |
| **Pornhub** | <https://pornhub.com><br><https://pornhubpremium.com> |
| **Rule34.xxx** | <https://rule34.xxx> |
| **Sankaku Complex** | <https://www.sankakucomplex.com><br><https://chan.sankakucomplex.com><br><https://idol.sankakucomplex.com> |
| **Soundcloud** | <https://soundcloud.com> |
| **小説家になろう** | <https://syosetu.com> |
| **TikTok** | <https://tiktok.com><br><https://douyin.com>|
| **TOKYO Motion** | <https://tokyomotion.net> |
| **Tumblr** | <https://tumblr.com> |
| **Twitch** | <https://twitch.tv> |
| **Twitter** | <https://twitter.com> |
| **Vimeo** | <https://vimeo.com> |
| **Wayback Machine** | <https://archive.org> |
| **Weibo** | <https://weibo.com> |
| **WikiArt** | <https://www.wikiart.org> |
| **xHamster** | <https://xhamster.com> |
| **XNXX** | <https://xnxx.com> |
| **XVideos** | <https://xvideos.com> |
| **Yande.re** | <https://yande.re> |
| **Youku** | <https://youku.com> |
| **YouTube** | <https://youtube.com> |
| **and more...** | [Supported sites by yt-dlp](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md) |
