<repo_name>puyangsky/DockerHubCrawler<file_sep>/dockerhub/items.py
# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class OfficialImageItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    name = scrapy.Field()
    link = scrapy.Field()
    linkmd5 = scrapy.Field()


class ClassifiedImageItem(scrapy.Item):
    repo_name = scrapy.Field()
    star_count = scrapy.Field()
    pull_count = scrapy.Field()
    short_description = scrapy.Field()
    is_automated = scrapy.Field()
    is_official = scrapy.Field()
    link = scrapy.Field()
    category = scrapy.Field()
<file_sep>/README.md
# DockerHubCrawler
Crawler of DockerHub using scrapy.
<file_sep>/dockerhub/pipelines.py
# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

from scrapy import signals
import json
import codecs
from twisted.enterprise import adbapi
from datetime import datetime
from hashlib import md5
import MySQLdb
import MySQLdb.cursors
import logging

logger = logging.getLogger()
handler = logging.FileHandler("/home/pyt/log/scrapy.log")
handler.setLevel(logging.WARN)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)


class BasePipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        dbargs = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['<PASSWORD>'],
            charset='utf8',
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=True,
        )
        dbpool = adbapi.ConnectionPool('MySQLdb', **dbargs)
        return cls(dbpool)

    # 获取url的md5编码
    def _get_linkmd5id(self, item):
        # url进行md5处理，为避免重复采集设计
        return md5(item['link']).hexdigest()

    # 异常处理
    def _handle_error(self, failue, item, spider):
        logger.err("err %s" % failue)


class OfficialImagePipeline(BasePipeline):

    # pipeline默认调用
    def process_item(self, item, spider):
        d = self.dbpool.runInteraction(self._do_upinsert, item, spider)
        d.addErrback(self._handle_error, item, spider)
        d.addBoth(lambda _: item)
        return d

    # 将每行更新或写入数据库中
    def _do_upinsert(self, conn, item, spider):
        linkmd5id = self._get_linkmd5id(item)
        conn.execute("select 1 from officialimage WHERE urlmd5 = %s", linkmd5id)
        ret = conn.fetchone()
        if ret:
            conn.execute("update officialimage set name = %s, url = %s, urlmd5 = %s",
                         (item['name'], item['link'], linkmd5id))
        else:
            conn.execute("insert into officialimage(name, url, urlmd5) values (%s, %s, %s)",
                         (item['name'], item['link'], linkmd5id))
            print "insert into officialimage(name, url, urlmd5) values (%s, %s, %s)", \
                (item['name'], item['link'], linkmd5id)


class ClassifiedImagePipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        dbargs = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['<PASSWORD>'],
            charset='utf8',
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=True,
        )
        dbpool = adbapi.ConnectionPool('MySQLdb', **dbargs)
        return cls(dbpool)

    def process_item(self, item, spider):
        d = self.dbpool.runInteraction(self._do_upinsert, item, spider)
        d.addErrback(self._handle_error, item, spider)
        d.addBoth(lambda _: item)
        return d

    def _do_upinsert(self, conn, item, spider):
        linkmd5id = self._get_linkmd5id(item)
        print linkmd5id
        conn.execute("select 1 from classifiedimage WHERE linkmd5 = %r", linkmd5id)
        ret = conn.fetchone()
        logger.warn("select 1 from classifiedimage WHERE linkmd5 = %r and ret is: %r" % (linkmd5id, ret))
        if not ret:
            # conn.execute("""
            # update classifiedimage set name = %s, star = %s, pull = %s, description = %s,
            #  automated = %s, official = %s, link = %s, linkmd5 = %s, category = %s
            # """, (item['repo_name'], item['star_count'], item['pull_count'], item['short_description'],
            #       item['is_automated'], item['is_official'], item['link'], linkmd5id, item['category']))
        # else:
            logger.warn("inserting one entry")
            try:
	            conn.execute("""
	            insert into classifiedimage(name, star, pull, description, automated,
	             official, link, linkmd5, category) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
	            """, (item['repo_name'], item['star_count'], item['pull_count'], item['short_description'],
	                  item['is_automated'], item['is_official'], item['link'], linkmd5id, item['category']))
	            logger.warn("insert successfully")
            except Exception, e:
            	logger.error("insert into mysql error: %r", e)

    # 获取url的md5编码
    def _get_linkmd5id(self, item):
        # url进行md5处理，为避免重复采集设计
        return md5(item['link']).hexdigest()

    # 异常处理
    def _handle_error(self, failue, item, spider):
        logger.error(failue)
<file_sep>/dockerhub/spiders/dockerhub_spider.py
import scrapy
from dockerhub.items import *
import json
from scrapy.http import Request
import time


class DockerhubSpider(scrapy.Spider):
    name = "dockerhub"
    allowed_domains = ["hub.docker.com"]
    base_url = "https://hub.docker.com/explore/?page="
    # base_query_url = "https://hub.docker.com/v2/search/repositories/?page=1&query=nginx&page_size=100"
    # https: // docker - index - static.s3.amazonaws.com / sitemap / sitemap.xml
    start_urls = []
    for i in range(1, 10):
        start_url = base_url + str(i)
        start_urls.append(start_url)

    def parse(self, response):
        items = []
        for sel in response.xpath('//ul/li'):
            item = OfficialImageItem()
            link = sel.xpath('a/@href').re("/_/.*")
            if link:
                # f.write("%s\n" % (link[0].split("/")[-2]))
                item['name'] = link[0].split("/")[-2]
                item['link'] = "https://hub.docker.com" + link[0]
                with open("/home/pyt/dockerhub/images", "a+") as f:
                    f.write("%s\n" % item['name'])
                # print item
                items.append(item)

        # return items


class SingleImageSpider(scrapy.Spider):
    name = "singleimage"
    allowed_domains = ["hub.docker.com"]
    # base_url = "https://hub.docker.com/explore/?page="
    base_query_url = "https://hub.docker.com/v2/search/repositories/?page=1&page_size=100&query="
    download_delay = 1
    start_urls = []
    for line in open("/home/pyt/dockerhub/images", "r"):
        start_url = base_query_url + line.strip("\n")
        start_urls.append(start_url)

    def parse(self, response):
        url = response.url
        # items = []
        json_str = response.body
        json_ob = json.loads(json_str)
        count = json_ob['count']
        print count
        results = json_ob['results']
        for result in results:
            # time.sleep(1)
            item = ClassifiedImageItem()
            item['star_count'] = result['star_count']
            item['pull_count'] = result['pull_count']
            item['short_description'] = result['short_description'].encode('utf-8')
            item['is_automated'] = result['is_automated']
            item['is_official'] = result['is_official']
            item['repo_name'] = result['repo_name'].encode('utf-8')
            if item['is_official']:
                item['link'] = "https://hub.docker.com/_/" + item['repo_name'].encode('utf-8')
            else:
                item['link'] = "https://hub.docker.com/r/" + item['repo_name'].encode('utf-8')
            item['category'] = url.split("query=")[1].split("&")[0].replace('%2', '/').encode('utf-8')
            yield item

        next_page_link = json_ob['next']
        # print next_page_link
        if next_page_link:
            yield Request(next_page_link, callback=self.parse)

    def parse_item(self, item):
        return item


