<repo_name>shiyang07ca/spider<file_sep>/README.md
## 爬虫练习

### 程序说明

编写两个线程对象分别维护两个队列，一个用于保存URL及深度，另一个负责将爬取到的内容存储至本地，
主函数在spider.py中负责初始化两个任务队列。

### 使用说明
终端输入
<code>python3 spider.py -u http://www.yingjiesheng.com -d 2</code>
将自动爬取不同深度的网页及链接（默认为2），输出内容保存至"Downloads"文件夹下，
格式为"A\[B]"，
A表示HTML的编号，B表示保存页面的URL深度。

### 参考链接
>1. https://www.metachris.com/2016/04/python-threadpool/
>2. https://www.ibm.com/developerworks/cn/aix/library/au-threadingpython/?ca=drs-tp3008#resources
>3. https://github.com/littlethunder/knowsecSpider2
<file_sep>/spider.py
#!/usr/bin/env python
'''
    Version: 0.1
'''

import time
import optparse
from queue import Queue

from CrawlerThread import CrawlerThread
from OutputThread import OutputThread


def parse_options():
    parser = optparse.OptionParser()

    parser.add_option("-u", "--url", dest="url", type="string")

    parser.add_option("-d", "--depth", dest="depth", type="int")

    options, _ = parser.parse_args()

    return options


# TODO: 编写主调度函数，封装该逻辑
def main(root_url, num_threads=10, depth=2):
    start = time.time()
    # 初始化两个任务队列，一个存储待爬取URL，一个用来存储待保存的内容
    url_queue = Queue()
    content_queue = Queue()

    for _ in range(num_threads):
        cr = CrawlerThread(url_queue, content_queue)
        cr.start()
    # 存入初始URL及深度的tuple
    url_queue.put((root_url, depth))
    for _ in range(num_threads):
        ot = OutputThread(content_queue)
        ot.start()

    # 阻塞线程直到任务完成
    try:
        url_queue.join()
        content_queue.join()
    except KeyboardInterrupt:
        print("Key Interrupt")
    else:
        print('****ALL DONE****')
    print("Elapsed Time: %s" % (time.time() - start))


if __name__ == "__main__":
    # root_url = "https://news.ycombinator.com/news"
    # root_url = "https://www.zhibo8.cc/"
    root_url = "http://www.yingjiesheng.com"
    options = parse_options()
    root_url = options.url or root_url
    depth = options.depth or 2

    main(root_url, depth=depth)
<file_sep>/OutputThread.py
import os
from threading import Thread


class OutputThread(Thread):
    def __init__(self, content_queue):
        Thread.__init__(self)
        self.content_queue = content_queue

    def run(self):
        """将内容队列取出并保存到本地"""
        while True:
            # 从content队列取出内容
            content_id, depth, content = self.content_queue.get()

            # 将id, 深度，content保存至本地HTML
            self._save_to_html(str(content_id) + '[' + str(depth) + ']', content)

            self.content_queue.task_done()

    @staticmethod
    def _save_to_html(filename, file):
        DEST_DIR = './Downloads/'
        path = os.path.join(DEST_DIR, filename+'.html')
        with open(path, 'wb') as f:
            if file:
                f.write(file)<file_sep>/CrawlerThread.py
import re
from threading import Thread

import requests
import bs4

class CrawlerThread(Thread):
    """从URL线程队列取出URL，将返回的响应存入内容队列"""
    def __init__(self, url_queue, content_queue):
        Thread.__init__(self)
        self.url_queue = url_queue  # URL队列
        self.content_queue = content_queue  # 内容队列
        self.content_id = 1  # 内容编号，用于给文件命名
        self.urls = set()  # 用于链接去重

    def run(self):
        while True:
            # 从url队列提取URL及深度，并请求页面内容
            url, depth = self.url_queue.get()
            content = self._request_data(url)

            # 将id,content放入队列
            self.content_queue.put((self.content_id, depth, content))
            self.content_id += 1

            # 如果深度大于零，提取页面链接并存入url_queue（深度为零说明到达最后一层，不再更新链接）
            if depth > 0:
                for link in self._extract_all_links(content):
                    if link not in self.urls:
                        self.urls.add(link)
                        self.url_queue.put((link, depth-1))

            self.url_queue.task_done()

    @staticmethod
    def _request_data(url):
        '''返回指定URL响应对象'''
        headers = {
            'Connection': 'keep-alive',
            'Accept': 'text/html,application/xhtml+xml,\
            application/xml;q=0.9,image/webp,*/*;q=0.8',
            'User-Agent': 'Mozilla/5.0 (X11; Linux i686)\
            AppleWebKit/537.36 (KHTML, like Gecko)\
            Chrome/35.0.1916.153 Safari/537.36',
             'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
        }
        if url is None:
            raise RuntimeWarning('url is None')
        try:
            resp = requests.get(url, headers=headers, timeout=2)
        except Exception as e:
            print(e)
        else:
            if resp.status_code != 200:
                resp.raise_for_status()
            return resp.content
        return None

    @staticmethod
    def _extract_all_links(content):
        '''返回页面内所有连接'''
        bs = bs4.BeautifulSoup(content, 'html.parser')
        links = []
        for item in bs.find_all('a', attrs={"href":re.compile(r'^(https?|www).*$')}):
            links.append(item.attrs['href'])
        # 将新的链接保存起来（测试用）
        with open('links.txt', 'a+') as f:
            for l in links:
                f.writelines(l)
                f.write('\n')
            f.write('*' * 20)
        return links