File: zhaopingou/zhaopingou_login.py

#!/usr/bin/ python3
# -*- coding: utf-8 -*-
import requests

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-04-06
"""

"""
模拟登陆招聘狗
"""

class ZhaoPinGouLogin(object):

    def __init__(self, account, password):
        self.url = "https://qiye.zhaopingou.com/zhaopingou_interface/security_login?timestamp=1554552162122"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",
            "Refer": "https://qiye.zhaopingou.com/signin?callback=https%3A%2F%2Fqiye.zhaopingou.com%2Fresume",
            "Host": "qiye.zhaopingou.com"
        }
        self.data = {
            'userName': account,
            'password': password,
            'code': '',
            'clientNo': '',
            'userToken': '',
            'clientType': '2'
        }
        self.session = requests.Session()

    def get_coolie(self):
        """模拟登陆获取cookie"""
        resp = self.session.post(
            url=self.url,
            headers=self.headers,
            data=self.data
        )
        resp_dict = resp.json()

        if resp_dict["errorCode"] == 1:
            print("登陆成功")
            # 获取登陆过的cookies
            cookies = resp.cookies
            print(cookies)
            return cookies
        else:
            print("登陆失败")

    def run(self):
        self.get_coolie()


if __name__ == '__main__':
    account = input("请输入你的账号：")
    password = input("请输入你的密码：")
    spider = ZhaoPinGouLogin(account, password)
    spider.run()



File: guoke/guoke.py

import requests
import re

headers_login = {
    'Accept-Language': 'zh-CN,zh;q=0.8',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Host': 'account.guokr.com',
    'Pragma': 'no-cache',
    'Cookie': '__utmt=1; __utma=253067679.2102330349.1540780238.1540780238.1541122809.2; __utmb=253067679.12.9.1541122812936; __utmc=253067679; __utmz=253067679.1540780238.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmv=253067679.|1=Is%20Registered=No=1; session=afcf1b0f-c71b-43d2-8046-f60ae28f9b45',
    'Referer': 'https://account.guokr.com/sign_in/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.61 Safari/537.36'
}

session = requests.Session()

url = 'https://account.guokr.com/sign_in/'
resp = session.get(url, headers=headers_login)
html = resp.text

csrf_token = re.search(r'id="csrf_token[\s\S]*?(\d+[\s\S]*?)"', html).group(1)
captcha_rand = re.search(r'id="captchaRand[\s\S]*?(\d+)', html).group(1)
img_url = 'https://account.guokr.com/captcha/' + captcha_rand
with open('captcha.jpg', 'wb') as fw:
    fw.write(session.get(img_url, headers=headers_login).content)

username = input('请输入用户名：')
password = input('请输入密码：')
captcha = input('请输入验证码 : ')

data = {
    'csrf_token': csrf_token,
    'username': username,
    'password': password,
    'captcha': captcha,
    'captcha_rand': captcha_rand,
    'permanent': 'y ',
}

response = session.post(url, data=data)
with open('response.html', 'w', encoding='utf-8') as fw:
    fw.write(response.text)

# print(response.cookies)
# print(session.cookies)

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.61 Safari/537.36'
}
homepage = 'https://www.guokr.com/i/0210199872/'
with open('homepage.html', 'w', encoding='utf-8') as fw:
    res = session.get(homepage, headers=headers)
    fw.write(res.text)



File: guoke/guoke_spider.py

# -*- coding: utf-8 -*-
import requests
from urllib.parse import urlencode
from requests import codes
import os
from multiprocessing.pool import Pool
from bs4 import BeautifulSoup as bsp
import json
import time
import re

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-7

"""


def get_index(offset):
    base_url = 'http://www.guokr.com/apis/minisite/article.json?'
    data = {
        'retrieve_type': "by_subject",
        'limit': "20",
        'offset': offset
    }
    url = base_url + urlencode(data)
    # print(url)
    try:
        resp = requests.get(url)
        if codes.ok == resp.status_code:
            return resp.json()
    except requests.ConnectionError:
        return None


# 解析出文章的url
def get_url(json):
    if json.get('result'):
        result = json.get('result')
        for item in result:
            if item.get('cell_type') is not None:
                continue
            yield item.get('url')
    """
    try:
        result=json.load(json)
        if result:
            for i in result.get('result'):
                yield i.get('url')
    """


# 解析文章详情页
def get_text(url):
    html = requests.get(url).text
    print(html)
    soup = bsp(html, 'lxml')
    title = soup.find('h1', id='articleTitle').get_text()
    autor = soup.find('div', class_="content-th-info").find('a').get_text()
    article_content = soup.find('div', class_="document").find_all('p')
    all_p = [i.get_text() for i in article_content if not i.find('img') and not i.find('a')]  # 去除标签
    article = '\n'.join(all_p)
    yield {"title": title, "autor": autor, "article": article}


def save_article(content):
    try:
        if content.get('title'):
            file_name = str(content.get('title')) + '.txt'
            with open(file_name, 'w', encoding='utf-8') as f:
                # f.write(json.dumps(content,ensure_ascii=False))
                f.write('\n'.join([str(content.get('title')), str(content.get('autor')), str(content.get('article'))]))
                print('Downloaded article path is %s' % file_name)
        else:
            file_name = str(content.get('title')) + '.txt'
            print('Already Downloaded', file_name)
    except requests.ConnectionError:
        print('Failed to Save Image，item %s' % content)


def main(offset):
    result = get_index(offset)
    all_url = get_url(result)
    for url in all_url:
        article = get_text(url)
        for art in article:
            # print(art)
            save_article(art)


GROUP_START = 0
GROUP_END = 7

if __name__ == '__main__':
    for i in range(GROUP_START, GROUP_END + 1):
        main(offset=i * 20 + 18)
        time.sleep(1)



File: 163email/163email.py

import time
from getpass import getpass
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait


def login():
    acount_num = input('请输入账号:')
    passwd_str = getpass('请输入密码:')
    driver = webdriver.Chrome()
    url = 'http://mail.163.com/'
    driver.get(url)

    # 等待页面加载完成，出现可以点击到密码登录的button
    wait = WebDriverWait(driver, 10)
    wait.until(EC.element_to_be_clickable((By.ID, 'lbNormal')))
    driver.find_element_by_id('lbNormal').click()


    # 使用CSSSelector正则匹配头部
    elem = driver.find_element_by_css_selector("iframe[id^='x-URS-iframe']")
    # 163登陆框是使用iframe进行嵌套的，所以需要先切换到该iframe
    driver.switch_to.frame(elem)

    account_el = driver.find_element_by_xpath('//input[@name="email"]')
    account_el.clear()
    account_el.send_keys(acount_num)

    password_el = driver.find_element_by_xpath('//input[@name="password"]')
    password_el.clear()
    password_el.send_keys(passwd_str)

    login_el = driver.find_element_by_xpath('//a[@id="dologin"]')
    login_el.click()

    time.sleep(10)
    cur_cookies = driver.get_cookies()
    return cur_cookies


if __name__ == '__main__':
    login()



File: lagou/Lagou.py

# -*- coding:utf-8 -*-
import re
import os
import time
import json
import sys
import subprocess
import requests
import hashlib
from bs4 import BeautifulSoup

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-6
"""


class Lagou_login(object):
    def __init__(self):
        self.session = requests.session()
        self.CaptchaImagePath = os.path.split(os.path.realpath(__file__))[0] + os.sep + 'captcha.jpg'
        self.HEADERS = {'Referer': 'https://passport.lagou.com/login/login.html',
                        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36'
                                      ' (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36'
                                      ' Core/1.53.4882.400 QQBrowser/9.7.13059.400',
                        'X-Requested-With': 'XMLHttpRequest'}

    # 密码加密
    def encryptPwd(self, passwd):
        # 对密码进行了md5双重加密
        passwd = hashlib.md5(passwd.encode('utf-8')).hexdigest()
        # veennike 这个值是在js文件找到的一个写死的值
        passwd = 'veenike' + passwd + 'veenike'
        passwd = hashlib.md5(passwd.encode('utf-8')).hexdigest()
        return passwd

    # 获取请求token
    def getTokenCode(self):
        login_page = 'https://passport.lagou.com/login/login.html'

        data = self.session.get(login_page, headers=self.HEADERS)

        soup = BeautifulSoup(data.content, "lxml", from_encoding='utf-8')
        '''
            要从登录页面提取token，code， 在头信息里面添加
            <!-- 页面样式 --><!-- 动态token，防御伪造请求，重复提交 -->
            <script type="text/javascript">
                window.X_Anti_Forge_Token = 'dde4db4a-888e-47ca-8277-0c6da6a8fc19';
                window.X_Anti_Forge_Code = '61142241';
            </script>
        '''
        anti_token = {'X-Anit-Forge-Token': 'None',
                      'X-Anit-Forge-Code': '0'}
        anti = soup.findAll('script')[1].getText().splitlines()
        anti = [str(x) for x in anti]

        anti_token['X-Anit-Forge-Token'] = re.findall(r'= \'(.+?)\'', anti[1])[0]
        anti_token['X-Anit-Forge-Code'] = re.findall(r'= \'(.+?)\'', anti[2])[0]

        return anti_token

    # 人工读取验证码并返回
    def getCaptcha(self):
        captchaImgUrl = 'https://passport.lagou.com/vcode/create?from=register&refresh=%s' % time.time()
        # 写入验证码图片
        f = open(self.CaptchaImagePath, 'wb')
        f.write(self.session.get(captchaImgUrl, headers=self.HEADERS).content)
        f.close()
        # 打开验证码图片
        if sys.platform.find('darwin') >= 0:
            subprocess.call(['open', self.CaptchaImagePath])
        elif sys.platform.find('linux') >= 0:
            subprocess.call(['xdg-open', self.CaptchaImagePath])
        else:
            os.startfile(self.CaptchaImagePath)

        # 输入返回验证码
        captcha = input("请输入当前地址(% s)的验证码: " % self.CaptchaImagePath)
        print('你输入的验证码是:% s' % captcha)
        return captcha

    # 登陆操作
    def login(self, user, passwd, captchaData=None, token_code=None):
        postData = {'isValidate': 'true',
                    'password': passwd,
                    # 如需验证码,则添加上验证码
                    'request_form_verifyCode': (captchaData if captchaData != None else ''),
                    'submit': '',
                    'username': user
                    }
        login_url = 'https://passport.lagou.com/login/login.json'

        # 头信息添加tokena
        login_headers = self.HEADERS.copy()
        token_code = self.getTokenCode() if token_code is None else token_code
        login_headers.update(token_code)

        # data = {"content":{"rows":[]},"message":"该帐号不存在或密码错误，请重新输入","state":400}
        response = self.session.post(login_url, data=postData, headers=login_headers)
        data = json.loads(response.content.decode('utf-8'))

        if data['state'] == 1:
            return response.content
        elif data['state'] == 10010:
            print(data['message'])
            captchaData = self.getCaptcha()
            token_code = {'X-Anit-Forge-Code': data['submitCode'], 'X-Anit-Forge-Token': data['submitToken']}
            return self.login(user, passwd, captchaData, token_code)
        else:
            print(data['message'])
            return False


if __name__ == "__main__":

    username = input("请输入你的手机号或者邮箱\n >>>:")
    passwd = input("请输入你的密码\n >>>:")

    lg = Lagou_login()
    passwd = lg.encryptPwd(passwd)

    data = lg.login(username, passwd)
    if data:
        print(data)
        print('登录成功')
    else:
        print('登录不成功')



File: baidu/util.py

#!/usr/bin/env python
# -*- coding:utf-8 -*-
import os
import json
from base64 import b64encode

from Crypto.PublicKey import RSA
from Crypto.Cipher import PKCS1_v1_5 as Cipher_pkcs1_v1_5


def encrypt_pwd(password, public_key):
    rsa_key = RSA.importKey(public_key)
    encryptor = Cipher_pkcs1_v1_5.new(rsa_key)
    cipher = b64encode(encryptor.encrypt(password.encode('utf-8')))
    return cipher.decode('utf-8')


def open_image(image_file):
    if os.name == "nt":
        os.system('start ' + image_file)  # for Windows
    else:
        if os.uname()[0] == "Linux":
            os.system("eog " + image_file)  # for Linux
        else:
            os.system("open " + image_file)  # for Mac


def save_image(resp, image_file):
    with open(image_file, 'wb') as f:
        for chunk in resp.iter_content(chunk_size=1024):
            f.write(chunk)


def parse_json(s):
    begin = s.find('{')
    end = s.rfind('}') + 1
    return json.loads(s[begin:end])



File: baidu/baidu.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import re
import sys
import time
from uuid import uuid4
from getpass import getpass

import requests

from util import *

if (sys.version_info < (3, 0)):
    input = raw_input


class BaiduLogin(object):
    def __init__(self):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.7 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.7',
            'referer': 'https://pan.baidu.com/',
        }
        self.sess = requests.session()
        self.gid = str(uuid4()).upper()
        self.token = None
        self.key = None
        self.public_key = None

    def _init_cookies(self):
        """初始化cookies
        :return:
        """
        self.sess.get(url='https://pan.baidu.com/', headers=self.headers)

    def _get_token(self):
        """获取登陆token
        :return:
        """
        url = 'https://passport.baidu.com/v2/api/?getapi'
        payload = {
            'getapi': '',
            'tpl': 'mn',
            'apiver': 'v3',
            'tt': str(int(time.time() * 1000)),
            'class': 'login',
            'gid': self.gid,
            'loginversion': 'v4',
            'logintype': 'dialogLogin',
            'traceid': '',
            'callback': 'bd__cbs__pivyke',
        }
        resp = self.sess.get(url=url, params=payload, headers=self.headers)
        js = parse_json(resp.text.replace("\'", "\""))
        self.token = js['data']['token']

    def _get_public_key(self):
        """获取RSA公钥
        :return: RSA公钥
        """
        url = 'https://passport.baidu.com/v2/getpublickey'
        payload = {
            'token': self.token,
            'tpl': 'mn',
            'apiver': 'v3',
            'tt': str(int(time.time() * 1000)),
            'gid': self.gid,
            'loginversion': 'v4',
            'traceid': '',
            'callback': 'bd__cbs__h02h0j'
        }
        resp = self.sess.get(url=url, params=payload, headers=self.headers)
        js = parse_json(resp.text.replace("\'", "\""))
        self.key, self.public_key = js.get('key'), js.get('pubkey')

    def login(self, username, password, retry=4):
        """用户名密码登陆
        :param username: 用户名
        :param password: 密码
        :return:
        """

        self._init_cookies()
        self._get_token()
        self._get_public_key()

        url = 'https://passport.baidu.com/v2/api/?login'
        data = {
            'staticpage': 'https://www.baidu.com/cache/user/html/v3Jump.html',
            'charset': 'UTF-8',
            'token': self.token,
            'tpl': 'netdisk',
            'subpro': 'netdisk_web',
            'apiver': 'v3',
            'tt': str(int(time.time() * 1000)),
            'codestring': '',
            'safeflg': '0',
            'u': 'https://www.baidu.com/',
            'isPhone': 'false',
            'detect': '1',
            'gid': self.gid,
            'quick_user': '0',
            'logintype': 'dialogLogin',
            'logLoginType': 'pc_loginDialog',
            'idc': '',
            'loginmerge': 'true',
            'splogin': 'rate',
            'username': username,
            'password': encrypt_pwd(password, self.public_key),
            'rsakey': self.key,
            'crypttype': '12',
            'ppui_logintime': 254896,
            'countrycode': '',
            'loginversion': 'v4',
            'traceid': '',
            'callback': 'parent.bd__pcbs__oxzeyj'
        }

        for _ in range(retry):
            resp = self.sess.post(url=url, headers=self.headers, data=data)

            m = re.search('.*href \+= "(.*)"\+accounts', resp.text)
            result = m.group(1)
            d = dict([x.split("=") for x in result.split("&")])

            err_no = d.get('err_no')
            if err_no == '0':
                print('Login success！')
                return
            elif err_no in ['6', '257']:
                code_string = d.get('codeString')
                data['codestring'] = code_string
                resp = self.sess.get(
                    url='https://passport.baidu.com/cgi-bin/genimage?{}'.format(code_string),
                    headers=self.headers
                )
                image_path = os.path.join(os.getcwd(), 'vcode-login.jpg')
                save_image(resp, image_path)
                open_image(image_path)
                verify_code = input('Please enter the verify code for login(return change):')
                data['verifycode'] = verify_code
            elif err_no == '120021':
                raise LoginError("Account is in risk, please do security verification first!")
            elif err_no in ['4', '7']:
                raise LoginError('Error username or password!')
            else:
                raise LoginError("Unknown error:" + result)

        raise LoginError('Login Fail！')


class LoginError(Exception):
    pass


if __name__ == '__main__':
    username = input("Username: ")
    password = getpass("Password: ")
    b = BaiduLogin()
    b.login(username=username, password=password)



File: bilibili/bilibili.py

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import TimeoutException
from PIL import Image
from io import BytesIO
from time import sleep
from getpass import getpass
import random

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-7
"""


class BiliBili():
    """
    登陆B站, 处理验证码
    电脑的缩放比例需要为100%, 否则验证码图片的获取会出现问题
    """

    def __init__(self, username, password):
        """
        初始化
        """
        options = webdriver.ChromeOptions()
        # 设置为开发者模式，避免被识别
        options.add_experimental_option('excludeSwitches',
                                        ['enable-automation'])
        self.browser = webdriver.Chrome(options=options)
        self.url = 'https://passport.bilibili.com/login'
        self.browser.get(self.url)
        self.wait = WebDriverWait(self.browser, 5, 0.2)
        self.username = username
        self.password = password

    def get_button(self):
        """
        获取滑动块, 并且返回
        :return: button
        """
        button = self.wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'gt_slider_knob')))
        return button

    def get_screenshot(self, button):
        """
        获取网页两次截图:
            1. 鼠标悬停于button的截图
            2. 鼠标点击button后的截图
        :param button: 滑动块
        :return: 两次截图的结果
        """
        ActionChains(self.browser).move_to_element(button).perform()
        screenshot1 = self.browser.get_screenshot_as_png()
        screenshot1 = Image.open(BytesIO(screenshot1))
        ActionChains(self.browser).click_and_hold(button).perform()
        screenshot2 = self.browser.get_screenshot_as_png()
        screenshot2 = Image.open(BytesIO(screenshot2))
        return (screenshot1, screenshot2)

    def get_position(self, button):
        """
        获取验证码图片的位置
        :return: 位置的四个点参数
        """
        ActionChains(self.browser).move_to_element(button).perform()
        img = self.wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'gt_box')))
        sleep(2)
        location = img.location
        size = img.size
        print(location, size)
        top, bottom, left, right = location['y'], location['y'] + size['height'], location['x'], \
                                   location['x'] + size['width']
        return top, bottom, left, right

    def get_geetest_image(self, button, name1='captcha1.png', name2='captcha2.png'):
        """
        获取两次验证码的截图:
            1. 鼠标悬停于button的截图
            2. 鼠标点击button后的截图
        :param button: 滑动块
        :param name1: 原始验证码保存的名字
        :param name2: 缺块验证码保存的名字
        :return: 两次验证码截图的结果
        """
        top, bottom, left, right = self.get_position(button)
        print('验证码位置', top, bottom, left, right)
        screenshot = self.get_screenshot(button)
        captcha1 = screenshot[0].crop((left, top, right, bottom))
        captcha1.save(name1)
        captcha2 = screenshot[1].crop((left, top, right, bottom))
        captcha2.save(name2)
        return (captcha1, captcha2)

    def login(self):
        """
        打开浏览器,并且输入账号密码
        :return: None
        """
        self.browser.get(self.url)
        username = self.wait.until(EC.element_to_be_clickable((By.ID, 'login-username')))
        password = self.wait.until(EC.element_to_be_clickable((By.ID, 'login-passwd')))
        sleep(1)
        username.send_keys(self.username)
        sleep(1)
        password.send_keys(self.password)

    def is_pixel_equal(self, img1, img2, x, y):
        """
        判断两个像素是否相同
        :param img1: 原始验证码
        :param img2: 缺块验证码
        :param x: 像素点的x坐标
        :param y: 像素点的y坐标
        :return: 像素是否相同
        """
        pixel1 = img1.load()[x-1, y]
        pixel2 = img2.load()[x-1, y]
        threshold = 100
        if abs(pixel1[0] - pixel2[0]) < threshold and abs(pixel1[1] - pixel2[1]) < threshold and abs(
                pixel1[2] - pixel2[2]) < threshold:
            return True
        else:
            return False

    def get_gap(self, img1, img2):
        """
        获取缺口偏移量
        :param img1: 原始验证码
        :param img2: 缺块验证码
        :return: 第二个缺块的左侧的x坐标
        """
        left = 60  # 大致忽略掉第一个缺块
        for i in range(left, img1.size[0]):
            for j in range(img1.size[1]):
                if not self.is_pixel_equal(img1, img2, i, j):
                    left = i
                    return left
        return left

    def get_track(self, distance):
        """
        获取滑块移动轨迹的列表
        :param distance: 第二个缺块的左侧的x坐标
        :return: 滑块移动轨迹列表
        """
        track = []
        current = 0
        mid = distance * 2 / 3
        t = 0.2
        v = 0
        distance += 10  # 使滑块划过目标地点, 然后回退
        while current < distance:
            if current < mid:
                a = random.randint(1, 3)
            else:
                a = -random.randint(3, 5)
            v0 = v
            v = v0 + a * t
            move = v0 * t + 0.5 * a * t * t
            current += move
            track.append(round(move))
        for i in range(2):
            track.append(-random.randint(2, 3))
        for i in range(2):
            track.append(-random.randint(1, 4))
        print(track)
        return track

    def move_button(self, button, track):
        """
        将滑块拖动到指定位置
        :param button: 滑动块
        :param track: 滑块运动轨迹列表
        :return: None
        """
        ActionChains(self.browser).click_and_hold(button).perform()
        for i in track:
            ActionChains(self.browser).move_by_offset(xoffset=i, yoffset=0).perform()
            sleep(0.0005)
        sleep(0.5)
        ActionChains(self.browser).release().perform()

    def crack(self):
        """
        串接整个流程:
            1. 输入账号密码
            2. 获取滑动块
            3. 获取两张验证码图片
            4. 获取滑块移动轨迹
            5. 将滑块拖动至指定位置
        :return:
        """
        self.login()
        button = self.get_button()
        captcha = self.get_geetest_image(button)
        left = self.get_gap(captcha[0], captcha[1])
        print(left)
        track = self.get_track(left)
        # 如果尝试登陆失败, 则重新验证, 最多三次
        times = 0
        while times < 3:
            self.move_button(button, track)
            try:
                success = self.wait.until(EC.text_to_be_present_in_element((By.CLASS_NAME, 'gt_info_type'), '验证通过:'))
                print(success)
            except TimeoutException as e:
                times += 1
                print('fail')
            else:
                print('success')
                return None


if __name__ == '__main__':
    ACCOUNT = input('请输入您的账号:')
    PASSWORD = getpass('请输入您的密码:')

    test = BiliBili(ACCOUNT, PASSWORD)  # 输入账号和密码
    test.crack()



File: douban/douban.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import requests

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-04-04
"""

"""
模拟登陆豆瓣
"""


class DouBanLogin(object):
    def __init__(self, account, password):
        self.url = "https://accounts.douban.com/j/mobile/login/basic"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
        }
        """初始化数据"""
        self.data = {
            "ck": "",
            "name": account,
            "password": password,
            "remember": "true",
            "ticket": ""
        }
        self.session = requests.Session()

    def get_cookie(self):
        """模拟登陆获取cookie"""
        html = self.session.post(
            url=self.url,
            headers=self.headers,
            data=self.data
        ).json()
        if html["status"] == "success":
            print("恭喜你，登陆成功")

    def get_user_data(self):
        """获取用户数据表明登陆成功"""
        # TODO: 这里填写你用户主页的url
        url = "这里填写你用户主页的url"
        # 获取用户信息页面
        html = self.session.get(url).text
        print(html)

    def run(self):
        """运行程序"""
        self.get_cookie()
        self.get_user_data()


if __name__ == '__main__':
    account = input("请输入你的账号:")
    password = input("请输入你的密码:")
    login = DouBanLogin(account, password)
    login.run()



File: douban/douban_spider.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
import json

import requests

# 定义请求url
url = "https://movie.douban.com/j/search_subjects"

# 定义请求头
headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
}
# 循环构建请求参数并且发送请求
for page_start in range(0, 100, 20):
    params = {
        "type": "movie",
        "tag": "热门",
        "sort": "recommend",
        "page_limit": "20",
        "page_start": page_start
    }
    response = requests.get(
        url=url,
        headers=headers,
        params=params
    )

    # 方式一 直接转换json方法
    # results = response.json()
    # 方式二：手动转换
    # 获取字符串
    content = response.content
    # 转换成字符串
    results = json.loads(content)

    # 解析结果
    for movie in results["subjects"]:
        print(movie["title"], movie["rate"])



File: jd_login/login_by_selenium.py

# tested on ubuntu15.04
import time
from selenium import webdriver

login_url = 'https://passport.jd.com/new/login.aspx'
driver = webdriver.PhantomJS()
driver.get(login_url)
time.sleep(5)

account = driver.find_element_by_id('loginname')
password = driver.find_element_by_id('nloginpwd')
submit = driver.find_element_by_id('loginsubmit')

account.clear()
password.clear()
account.send_keys('yourname')
password.send_keys('yourpassword')

submit.click()
time.sleep(5)

# cookie和前面一样的方式获取和保存
cookies = driver.get_cookies()
driver.close()



File: jd_login/Method_Second/Config.py

"""
Config.py
配置文件
"""

settings = {
    #一天申请的限制个数
    'maxApplyNum' : 300 ,
    #试用类型
    #家用电器737 手机数码652 电脑办公670 家居家装1620 服饰鞋包1315 生鲜美食12218 钟表奢品5025 家庭清洁15901 食品饮料1320
    'cids' : ['737', '652' ,'670', '1620', '1315', '12218' ,'5025' , '15901' ,'1320' ,] ,
    #申请商品价格下限 单位 元
    'goodPrice' : 30 ,
    #浏览器button最长等待时间 单位秒
    'waitTime' : 10 ,
    #试用结束后是否自动关机 True代表关机
    'shutdown' : False ,
}



File: jd_login/Method_Second/main.py

"""
京东试用自动申请程序，每天仅需执行一次即可
"""

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support import  expected_conditions as EC
from selenium.webdriver.common.by import By
import time
from pyquery import PyQuery as pq
import json
import os
import getpass
import base64

#载入自己编写的配置文件
from Config import settings

#全局变量

#打开无界面的chrome浏览器
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
#不打印不重要的日志信息
chrome_options.add_argument('log-level=2')
browser = webdriver.Chrome(chrome_options = chrome_options)
#设置浏览器最长等待时间
wait = WebDriverWait(browser, settings['waitTime'])

#打开用于登陆的chrome浏览器
browser_login = webdriver.Chrome()
#设置浏览器最长等待时间
wait_login = WebDriverWait(browser_login, settings['waitTime'])

def readCookies():
    """
    从文件中读取cookies并返回 文件不存在则返回False
    """
    #不存在cookies文件
    if os.path.exists("cookies.json") == False:
        print("cookies文件不存在！")
        return False
    with open("cookies.json","r") as f:
        cookies = json.load(f)
    return cookies

def writeCookies(cookies):
    """
    从浏览器中向文件写入cookies
    """
    with open("cookies.json", "w") as f:
        json.dump(cookies, f)

def closeSW(iApplyNum):
    """
    在文件中输出申请个数 iApplyNum
    关闭了浏览器和程序
    """
    #等待5秒
    time.sleep(5)
    #保存浏览器cookies到文件中
    cookies = browser.get_cookies()
    writeCookies(cookies)

    #关闭浏览器
    browser.quit()
    with open("log.txt", 'a') as f:
        #输出申请时间和数量
        f.write( time.ctime() + " 申请数量：" + str(iApplyNum) + '\n')

    #是否关闭电脑
    if settings['shutdown'] == True:
        os.system("shutdown -s -f")

    #退出程序
    exit()

def genekeys():
    #打开正确/屏蔽词文件,并处理
    keys = []
    for line in open("Truekeyword.txt", 'r' ,encoding='UTF-8' ):
        line = line[0:line.find('\n')]
        if line == '':
            continue
        line = line.split('/')
        line[0] = line[0].strip()
        line[1] = line[1].strip()
        if line[0] == '':
            line[0] = []
        else:
            line[0] = line[0].split(' ')
        if line[1] == '':
            line[1] = []
        else:
            line[1] = line[1].split(' ')
        keys.append(line)
    return keys

def goodJudge(goodName, goodPrice, keys):
    """
    根据商品名称和价格判断是否试用该商品
    """
    if goodPrice < settings['goodPrice']:
        return False

    for key in keys:
        booltrue = False
        if key[0] == []:
            booltrue = True
        for tk in key[0]:
            if tk == '':
                continue
            if tk in goodName:
                booltrue = True
                break
        if booltrue == False:
            continue
        for tk in key[1]:
            if tk == '':
                continue
            if tk in goodName:
                return False
    return True

def do_try(url):
    """
    对于某个商品申请试用
    url为申请网址 iApplyNum为当前申请成功的个数
    """
    try:
        #切换到选项卡1
        browser.switch_to.window(browser.window_handles[1])
        #访问商品网页
        browser.get(url)
        #停2秒
        time.sleep(2)

        #获取网页的html源码
        html = browser.page_source

        #初始化pyquery
        doc = pq(html)

        #获取申请试用的botton
        button = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR,'#product-intro > div.info > div.try-info.clearfix.bigImg > div.info-detail.chosen > div > div.btn-wrap > a'))
        )
        #如果上面写的不是申请试用，就申请下一个
        if button.text!='申请试用':
            return False
        #点击申请试用
        button.click()
        #找到关注并申请的按钮
        button2 = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR,'body > div.ui-dialog > div.ui-dialog-content > div > div > div.btn > a.y'))
        )
        
        time.sleep(1)
        #点击关注
        button2.click()
        #此时试用一件商品完成
        time.sleep(2)
        return True
    #抛出超时异常
    except TimeoutException:
        #这件商品不申请了，返回
        return False


def get_try(cid, iApplyNum, maxApplyNum, keys):
    browser.get('https://try.jd.com/activity/getActivityList?page=1&cids='+cid)

    #获取网页的html源码
    html = browser.page_source

    #初始化pyquery
    doc = pq(html)

    #CSS选择器 找出总页数
    pageitem = doc('.root61 .container .w .p-wrap .p-skip').items()
    #为了应对命名空间而采用的粗暴办法
    pagestr = list(pageitem)[0].text()
    pagestr = pagestr[2:]
    pagestr = pagestr[0:pagestr.find('\n')]
    pagenum = int(pagestr)
    print("商品总页数：" + str(pagenum+1) )

    for i in range(pagenum):

        if i >=1:
            #切换到下一页
            browser.get('https://try.jd.com/activity/getActivityList?page='+str(i+1)+'&cids='+cid)
            #停2秒
            time.sleep(2)
        html = browser.page_source
        doc = pq(html)
        #CSS选择器 找出商品列表
        items = doc('.root61 .container .w .goods-list .items .con .clearfix .item').items()
        
        #迭代器转换为list类型
        items=list(items)
        
        #对于每个商品进行处理
        for item in items:
            #按钮为已申请
            if item('.try-item .try-button').text() == '已申请':
                #已经申请过的不申请
                continue
            #商品名称
            itemname = item('.try-item .p-name').text()
            #商品价格
            itempricetext = item('.try-item .p-price').text()
            #截取多余的文本
            #找不到价格 出现暂无报价的情况
            if itempricetext.find('￥') == -1:
                itemprice = 0
            else:
                itempricetext = itempricetext[itempricetext.find('￥')+1:]
                #goodPrice 商品价格
                itemprice = float(itempricetext)
            if goodJudge(itemname, itemprice, keys) == False:
                #不申请了
                continue
            
            itemurl = item('.try-item .link')
            #试用该商品
            if do_try('https:'+itemurl.attr('href')) == True:
                print("申请成功 " +str(itemprice) + "  " + itemname)
                iApplyNum = iApplyNum + 1
            #停3秒
            time.sleep(2)
            browser.switch_to.window(browser.window_handles[0])

            if iApplyNum >= maxApplyNum:
                print("已经成功申请" + str(maxApplyNum) + "件商品 申请结束")
                closeSW(iApplyNum)
            time.sleep(2)
        print(cid+'类:第'+str(i+1)+'页申请完成')
    return iApplyNum


def trycid():
    """
    控制申请类别和数量 返回已申请数量iApplyNum
    """
    keys = genekeys()
    #京东限制 每天最大申请数量
    maxApplyNum = settings['maxApplyNum']
    iApplyNum = 0
    #获取试用类型
    cids = settings['cids']
    browser.get('https://try.jd.com/')
    browser.get('https://try.jd.com/activity/getActivityList')
    #执行js脚本 打开一个新选项卡
    browser.execute_script('window.open()')
    browser.switch_to.window(browser.window_handles[0])
    for cid in cids:
        iApplyNum = get_try(cid, iApplyNum, maxApplyNum, keys)
    return iApplyNum

def login():
    """
    登陆函数
    """
    #必须访问一次京东
    browser_login.get('https://jd.com')
    #读取文件中的cookies
    cookies = readCookies()
    if cookies != False:
        #如果从文件中读取到了cookies，就放入浏览器中
        for cookie in cookies:
            browser_login.add_cookie(cookie)
    #直接去登陆界面
    browser_login.get('https://passport.jd.com/login.aspx')
    #找到账户登陆的窗口
    button_login = browser_login.find_elements_by_css_selector('#content > div.login-wrap > div.w > div > div.login-tab.login-tab-r > a')
    button_login = button_login[0]
    #点击
    button_login.click()
    time.sleep(2)

    #取得用户名和密码的过程
    #如果文件不存在
    if os.path.exists("login.txt") == False:
        username = input("请输入京东用户名:")
        password = getpass.getpass("请输入京东密码(输入不会显示在屏幕上):")
    else:
        #从文件中读入用户名和密码
        with open("login.txt",) as f:
            up = f.read()
        up = up.split('\n')
        username = up[0].encode()
        password = up[1].encode()
        #base64解码
        username = base64.b64decode(username)
        username = username.decode()
        password = base64.b64decode(password)
        password = password.decode()

    #找到输入框
    input_username =  browser_login.find_element_by_name('loginname')
    #输入用户名
    input_username.send_keys(username)
    #找到密码框
    input_password = browser_login.find_element_by_name('nloginpwd')
    #输入密码
    input_password.send_keys(password)
    #找到登录按钮
    button_logOK = browser_login.find_elements_by_id('loginsubmit')
    button_logOK = button_logOK[0]
    time.sleep(2)
    #点击
    button_logOK.click()

    #循环检测是否登陆
    while 1:
        try:
            wait_login.until(
                EC.presence_of_element_located((By.CSS_SELECTOR,
                                        '#ttbar-login > div.dt.cw-icon > a'))
            )
            break
        except TimeoutException:
            continue
    print('登陆成功！')
    time.sleep(2)

    #登录成功后 若不存在login.txt，则把用户名和密码写入文件
    if os.path.exists("login.txt") == False:
        #base64编码
        username = username.encode()
        username = base64.b64encode(username)
        password = password.encode()
        password = base64.b64encode(password)
        # 写入文件中
        with open("login.txt", "w") as f:
            f.write(username.decode() +"\n")
            f.write(password.decode())

    #把登陆浏览器的cookie转移到无界面浏览器上
    #取得原浏览器的所有cookie
    cookies = browser_login.get_cookies()
    browser.get('https://www.jd.com')
    #cookies是一个以字典为元素的list
    for cookie in cookies:
        browser.add_cookie(cookie)
    #关闭登陆浏览器
    browser_login.quit()

if __name__ == '__main__':

    #登陆
    login()
    #开始申请 iApplyNum为申请成功的个数
    iApplyNum = trycid()
    #申请结束
    closeSW(iApplyNum)



File: jd_login/Method_First/config.py

#coding:utf-8

settings = {
    'auto_shutdown':False,  #是否自动关机，默认为False
    'total_products':300,   #要申请的商品个数上限，默认为300
    'total_num_of_page':50, #申请前total_num_of_page页
    'choice':False,      #是否按照商品名称选择要申请的商品，如果设置为True，则应该创建choice.txt文件
                        #并将想要的商品名称写进去即可。默认为False
    'ban':False          #是否按照商品名称选择要过滤掉的商品，如果设置为True，则应该创建ban.txt文件
                        #并将想过滤掉的商品名称写进去即可。(不同商品名称之间用,.!空格或换行符隔开即可)默认为False
}



File: jd_login/Method_First/Try_selenium.py

#coding=utf-8
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support import  expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time
import os
from pyquery import PyQuery as pq
from config import settings as SET
import re

#browser_for_login为正常浏览器，用于登录
browser_for_login = webdriver.Chrome()

chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
#无头模式
browser = webdriver.Chrome(chrome_options=chrome_options)
wait = WebDriverWait(browser,10)

total_num_of_products = SET['total_products']
total_num_of_products_cur = 0

choice_list=[]
ban_list=[]



#所有的sleep为了是减慢速度, 防止被检查异常
def do_try(url):
    try:
        browser.switch_to.window(browser.window_handles[1])
        browser.get(url)
        time.sleep(2)
        button = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR,'#product-intro > div.info > div.try-info.clearfix.bigImg > div.info-detail.chosen > div > div.btn-wrap > a'))
        )
        #如果按钮不是‘申请使用’，则说明该商品申请出错或者已经申请过了，则跳回到试用商品列表界面
        if button.text!='申请试用':
            browser.switch_to.window(browser.window_handles[0])
            return False
        button.click()
        #等待关注商铺的信息出来，然后点击关注即可。如果无需关注，则可能会抛出超时异常
        button2 = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR,'body > div.ui-dialog > div.ui-dialog-content > div > div > div.btn > a.y'))
        )
        time.sleep(1)
        button2.click()

        time.sleep(2)
        browser.switch_to.window(browser.window_handles[0])
        return True
    except TimeoutException:
        browser.switch_to.window(browser.window_handles[0])     #抛出超时异常则返回到试用商品列表界面即可
        return True


def get_try(page):
    url='https://try.jd.com/activity/getActivityList'+'?page='+str(page)
    browser.get(url)

    time.sleep(2)

    html = browser.page_source
    #print(html)

    #利用PyQuery获得所有关于试用商品跳转的class=item的<li>标签
    doc = pq(html)
    #因为已经申请过的商品的<li>标签中的class除了item，还有applied，故将其删除之后申请便可跳过已申请的商品
    doc('.applied').remove()
    items = doc('.root61 .container .w .goods-list .items .con .clearfix .item').items()
    #print(type(items))
    #print(items)
    items=list(items)

    for item in items:
        #获得每个商品的标题，如果进行商品过滤则有可能有用
        title = item('.p-name').text()

        if check_name(title) == False:
            continue

        price_text = item('.p-price').text()[6:]
        price = float(price_text)
        if price < float(SET['price_limit']):
            continue
        try_url = 'https:'+item('.link').attr('href')
        print('价格: ',price)
        print(title)
        #print(try_url)
        time.sleep(1)
        global total_num_of_products_cur
        global total_num_of_products
        if do_try(try_url) == True:
            total_num_of_products_cur +=1
            print("申请成功")
            print('')
        else :
            print("申请失败")
            print('')

        #到达指定个数之后退出
        if total_num_of_products_cur >= total_num_of_products:
            return



def Control_try(total_page):
    browser.execute_script('window.open()')
    browser.switch_to.window(browser.window_handles[0])
    for page in range(1,total_page+1):
        print('开始申请第'+str(page)+'页')
        get_try(page)
        global total_num_of_products
        global total_num_of_products_cur
        if total_num_of_products_cur >= total_num_of_products:
            return
        print('第'+str(page)+'页申请完成')

#成功登录后将browser_for_login的cookies取出放到无头browser中即可
def login():
    browser_for_login.get('https://passport.jd.com/new/login.aspx')
    while browser_for_login.current_url!='https://www.jd.com/':
        time.sleep(2)

    cookies = browser_for_login.get_cookies()
    browser_for_login.close()
    browser.get('https://www.jd.com')
    for cookie in cookies:
        browser.add_cookie(cookie)

    browser.get('https://www.jd.com')

def auto_showdown():
    if SET['auto_shutdown'] == True:
        print('\n5秒后将自动关机')
        time.sleep(5)
        os.system('shutdown -s -t 1')

def deal_file():
    global choice_list
    global ban_list
    if SET['choice']==True:
        with open('choice.txt','r') as f:
            choice_list = re.split('[ |.|,|!|\n]',f.read())
            f.close()

    if SET['ban']==True:
        with open('ban.txt','r') as f:
            ban_list = re.split('[ |.|,|!|\n]',f.read())
            f.close()

def check_name(title):
    is_choice = False
    if len(choice_list)==0:
        is_choice = True
    for ch in choice_list:
        if ch in title:
            is_choice = True
            break

    if is_choice == False:
        return False
    is_ban = False
    for ba in ban_list:
        if ba in title:
            is_ban = True
            break

    if is_ban == True:
        return False
    return True


if __name__ == '__main__':
    deal_file()
    login()
    #申请前SET['total_num_of_page']页
    Control_try(SET['total_num_of_page'])
    browser.close()
    print('申请完成')
    auto_showdown()



File: baidu_translate/Baidufanyi.py

# !/usr/bin/python3
# -*- coding: utf-8 -*-

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
"""

"""
请求url分析 :https://fanyi.baidu.com/basetrans
请求方式分析 :POST
请求参数分析 : {
query: hello
from: en
to: zh
token: 6f5c83b84d69ad3633abdf18abcb030d
sign: 54706.276099}
请求头分析
"""

# 代码实现流程
# 1. 实现面对对象构建爬虫对象
# 2. 爬虫流程四步骤
# 2.1 获取URl
# 2.2 发送请求获取响应
# 2.3 从响应中提取数据
# 2.4 保存数据

import requests
import js2py

context = js2py.EvalJs()

# 翻译模式
# 0:英译中 1:中译英
translating_mode = 0

class BaiDuTranslater(object):
    """
    百度翻译爬虫
    """

    def __init__(self, query):
        # 初始化
        self.url = "https://fanyi.baidu.com/basetrans"
        self.query = query
        self.headers = {
            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
            "Referer": "https://fanyi.baidu.com/",
            "Cookie": "BAIDUID=714BFAAF02DA927F583935C7A354949A:FG=1; BIDUPSID=714BFAAF02DA927F583935C7A354949A; PSTM=1553390486; delPer=0; PSINO=5; H_PS_PSSID=28742_1463_21125_18559_28723_28557_28697_28585_28640_28604_28626_22160; locale=zh; from_lang_often=%5B%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%2C%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%5D; to_lang_often=%5B%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%2C%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%5D; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH=1; Hm_lvt_afd111fa62852d1f37001d1f980b6800=1553658863,1553766321,1553769980,1553770442; Hm_lpvt_afd111fa62852d1f37001d1f980b6800=1553770442; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1553766258,1553766321,1553769980,1553770442; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1553770442"
        }

    def make_sign(self):
        # js逆向获取sign的值
        with open("translate.js", "r", encoding="utf-8") as f:
            context.execute(f.read())

        # 调用js中的函数生成sign
        sign = context.a(self.query)
        # 将sign加入到data中
        return sign

    def make_data(self, sign):
        # 判断翻译模式,选取对应的 from 和 to 值.
        if translating_mode == 0:
            from_str = "en"
            to_str = "zh"
        else:
            from_str = "zh"
            to_str = "en"
        data = {
            "query": self.query,
            "from": from_str,
            "to": to_str,
            "token": "6f5c83b84d69ad3633abdf18abcb030d",
            "sign": sign
        }
        return data

    def get_content(self, data):
        # 发送请求获取响应
        response = requests.post(
            url=self.url,
            headers=self.headers,
            data=data
        )
        return response.json()["trans"][0]["dst"]

    def run(self):
        """运行程序"""
        # 获取sign的值
        sign = self.make_sign()
        # 构建参数
        data = self.make_data(sign)
        # 获取翻译内容
        content = self.get_content(data)
        print(content)


if __name__ == '__main__':
    translating_mode = int(input("请输入翻译模式(0:英译中 1:中译英):"))
    query = input("请输入您要翻译的内容:")
    translater = BaiDuTranslater(query)
    translater.run()



File: qqmusic/qqmusic_spider.py

# -*- coding: utf-8 -*-
# @Author: MediocrityXT
# @Github: https://github.com/MediocrityXT


import requests
import execjs
import os

class Spider(object):
    def __init__(self):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0',
            'Referer':'https://y.qq.com/portal/player.html'
        }
    
    def __get_songs(self, name):
        num = 10        
        url = 'https://c.y.qq.com/soso/fcgi-bin/client_search_cp?p=1&n='+str(num)+'&w='+name+'&format=json'
        response = requests.get(url, headers=self.headers).json()['data']['song']['list']
        return response

    def __print_info(self, songs):
        #打印搜索到的的歌曲信息
        index = 0
        for it in songs:
            index = index + 1
            if it['pay']['payplay']:
                NeedPay='(收费)  ' 
            else:
                NeedPay='        ' 
            singers=''
            it2=it['singer']
            for x in it2:
                singers = singers+x['name']+' '
            print(index,'.',NeedPay,it['songname'],'     ',singers)

    def __get_Sign(self,data):
        print(os.path.realpath(__file__))
        with open(os.path.realpath(__file__)+'/../sign.js', 'r', encoding='utf-8') as f:
            js_content = f.read()
        js_exec = execjs.compile(js_content)
        sign = js_exec.call('getSecuritySign',data)
        return sign

    def __get_url(self,data):
        sign=self.__get_Sign(data)
        
        url='https://u.y.qq.com/cgi-bin/musics.fcg?-=getplaysongvkey38596056557178904'\
        '&g_tk=1129808082'\
        '&sign={}'\
        '&loginUin=18585073516'\
        '&hostUin=0'\
        '&format=json'\
        '&inCharset=utf8'\
        '&outCharset=utf-8'\
        '&notice=0'\
        '&platform=yqq.json'\
        '&needNewCode=0&data='.format(sign)+data
        response=requests.get(url).json()
        return response['req_0']['data']['midurlinfo'][0]['purl']

    def __set_data(self, songmid):
        data='{"req":{"module":"CDN.SrfCdnDispatchServer","method":"GetCdnDispatch","param":{"guid":"358840384","calltype":0,"userip":""}},'\
        '"req_0":{"module":"vkey.GetVkeyServer","method":"CgiGetVkey","param":{"guid":"358840384","songmid":["'+songmid+'"],"songtype":[0],"uin":"18585073516","loginflag":1,"platform":"20"}},"comm":{"uin":18585073516,"format":"json","ct":24,"cv":0}}'
        return data

    def __download_mp3(self, url, filename):
        abspath = os.path.abspath('.')  # 获取绝对路径
        os.chdir(abspath)
        response = requests.get(url, headers=self.headers).content
        path = os.path.join(abspath, filename)
        with open(path + '.m4a', 'wb') as f:
            f.write(response)
            print('下载完毕,保存至:%s.m4a' % path )

    def run(self):
        while True:
            name = input('搜索歌曲名称：')
            songs = self.__get_songs(name)
            self.__print_info(songs)
            choice = int(input('请输入左边对应数字选择歌曲:'))-1
            if choice >=0 & choice<len(songs):
                if songs[choice]['pay']['payplay']:
                    print('无法下载收费歌曲')
                else:
                    songmid=songs[choice]['songmid']
                    data=self.__set_data(songmid)
                    url='https://isure.stream.qqmusic.qq.com/'+self.__get_url(data)
                    #print(url)
                    self.__download_mp3(url,songs[choice]['songname'])
            else:
                print('输入错误')
            
            flag = input('如需继续可以按任意键进行搜歌，否则按0结束程序')
            if flag == '0':
                break
        print('程序结束！')
        
if __name__=='__main__':
    spider = Spider()
    spider.run()



File: webWeixin/webWeixin.py

import os
import re
import time
import sys
import subprocess
import requests
import xml.dom.minidom
import json

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-6
"""

session = requests.session()
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}

QRImgPath = os.path.split(os.path.realpath(__file__))[0] + os.sep + 'webWeixinQr.jpg'
uuid = ''
tip = 0
base_uri = ''
redirect_uri = ''
skey = ''
wxsid = ''
wxuin = ''
pass_ticket = ''
deviceId = 'e000000000000000'
BaseRequest = {}
ContactList = []
My = []
SyncKey = ''


def getUUID():
    global uuid, session

    url = 'https://login.weixin.qq.com/jslogin'
    params = {
        'appid': 'wx782c26e4c19acffb',
        'fun': 'new',
        'lang': 'zh_CN',
        '_': int(time.time()),
    }

    response = session.get(url, params=params)
    data = response.content.decode('utf-8')
    # print(data) >>> window.QRLogin.code = 200; window.QRLogin.uuid = "oZwt_bFfRg==";

    regx = r'window.QRLogin.code = (\d+); window.QRLogin.uuid = "(\S+?)"'
    pm = re.search(regx, data)

    code = pm.group(1)
    uuid = pm.group(2)

    if code == '200':
        return True

    return False


def showQRImage():
    global tip

    url = 'https://login.weixin.qq.com/qrcode/' + uuid
    params = {
        't': 'webwx',
        '_': int(time.time()),
    }

    response = session.get(url, params=params)

    tip = 1

    with open(QRImgPath, 'wb') as f:
        f.write(response.content)
        f.close()

    if sys.platform.find('darwin') >= 0:
        subprocess.call(['open', QRImgPath])
    elif sys.platform.find('linux') >= 0:
        subprocess.call(['xdg-open', QRImgPath])
    else:
        os.startfile(QRImgPath)

    print('请使用微信扫描二维码以登录')


def waitForLogin():
    global tip, base_uri, redirect_uri

    url = 'https://login.weixin.qq.com/cgi-bin/mmwebwx-bin/login?tip=%s&uuid=%s&_=%s' % (
        tip, uuid, int(time.time()))

    response = session.get(url)
    data = response.content.decode('utf-8')

    # print(data)

    # window.code=500;
    regx = r'window.code=(\d+);'
    pm = re.search(regx, data)

    code = pm.group(1)

    if code == '201':  # 已扫描
        print('成功扫描,请在手机上点击确认以登录')
        tip = 0
    elif code == '200':  # 已登录
        print('正在登录...')
        regx = r'window.redirect_uri="(\S+?)";'
        pm = re.search(regx, data)
        redirect_uri = pm.group(1) + '&fun=new'
        base_uri = redirect_uri[:redirect_uri.rfind('/')]

        # closeQRImage
        if sys.platform.find('darwin') >= 0:  # for OSX with Preview
            os.system("osascript -e 'quit app \"Preview\"'")
    elif code == '408':  # 超时
        pass
    # elif code == '400' or code == '500':

    return code


def login():
    global skey, wxsid, wxuin, pass_ticket, BaseRequest

    response = session.get(redirect_uri)
    data = response.content.decode('utf-8')
    doc = xml.dom.minidom.parseString(data)
    root = doc.documentElement

    for node in root.childNodes:
        if node.nodeName == 'skey':
            skey = node.childNodes[0].data
        elif node.nodeName == 'wxsid':
            wxsid = node.childNodes[0].data
        elif node.nodeName == 'wxuin':
            wxuin = node.childNodes[0].data
        elif node.nodeName == 'pass_ticket':
            pass_ticket = node.childNodes[0].data

    # print('skey: %s, wxsid: %s, wxuin: %s, pass_ticket: %s' % (skey, wxsid,
    # wxuin, pass_ticket))

    if not all((skey, wxsid, wxuin, pass_ticket)):
        return False

    BaseRequest = {
        'Uin': int(wxuin),
        'Sid': wxsid,
        'Skey': skey,
        'DeviceID': deviceId,
    }

    return True


def webwxinit():
    url = base_uri + \
          '/webwxinit?pass_ticket=%s&skey=%s&r=%s' % (
              pass_ticket, skey, int(time.time()))
    params = {
        'BaseRequest': BaseRequest
    }

    h = headers
    h['ContentType'] = 'application/json; charset=UTF-8'
    response = session.post(url, data=json.dumps(params), headers=h)
    data = response.content.decode('utf-8')
    # print(data)

    global ContactList, My, SyncKey

    dic = json.loads(data)
    ContactList = dic['ContactList']
    My = dic['User']

    SyncKeyList = []
    for item in dic['SyncKey']['List']:
        SyncKeyList.append('%s_%s' % (item['Key'], item['Val']))
    SyncKey = '|'.join(SyncKeyList)

    # ErrMsg = dic['BaseResponse']['ErrMsg']

    Ret = dic['BaseResponse']['Ret']
    if Ret != 0:
        return False

    return True


def webwxgetcontact():
    url = base_uri + \
          '/webwxgetcontact?pass_ticket=%s&skey=%s&r=%s' % (
              pass_ticket, skey, int(time.time()))

    h = headers
    h['ContentType'] = 'application/json; charset=UTF-8'
    response = session.get(url, headers=h)
    data = response.content.decode('utf-8')
    # print(data)

    dic = json.loads(data)
    MemberList = dic['MemberList']

    # 倒序遍历,不然删除的时候出问题..
    SpecialUsers = ["newsapp", "fmessage", "filehelper", "weibo", "qqmail", "tmessage", "qmessage", "qqsync",
                    "floatbottle", "lbsapp", "shakeapp", "medianote", "qqfriend", "readerapp", "blogapp", "facebookapp",
                    "masssendapp",
                    "meishiapp", "feedsapp", "voip", "blogappweixin", "weixin", "brandsessionholder", "weixinreminder",
                    "wxid_novlwrv3lqwv11", "gh_22b87fa7cb3c", "officialaccounts", "notification_messages", "wxitil",
                    "userexperience_alarm"]
    for i in range(len(MemberList) - 1, -1, -1):
        Member = MemberList[i]
        if Member['VerifyFlag'] & 8 != 0:  # 公众号/服务号
            MemberList.remove(Member)
        elif Member['UserName'] in SpecialUsers:  # 特殊账号
            MemberList.remove(Member)
        elif Member['UserName'].find('@@') != -1:  # 群聊
            MemberList.remove(Member)
        elif Member['UserName'] == My['UserName']:  # 自己
            MemberList.remove(Member)

    return MemberList


def main():
    if not getUUID():
        print('获取uuid失败')
        return

    showQRImage()
    time.sleep(1)

    while waitForLogin() != '200':
        pass

    os.remove(QRImgPath)

    if not login():
        print('登录失败')
        return
    # 登录完成, 下面查询好友
    if not webwxinit():
        print('初始化失败')
        return

    MemberList = webwxgetcontact()

    print('通讯录共%s位好友' % len(MemberList))

    for x in MemberList:
        sex = '未知' if x['Sex'] == 0 else '男' if x['Sex'] == 1 else '女'
        print('昵称:%s, 性别:%s, 备注:%s, 签名:%s' % (x['NickName'], sex, x['RemarkName'], x['Signature']))


if __name__ == '__main__':
    print('开始')
    main()



File: tuchong/tuchong.py

import requests
import re
import os
from hashlib import md5
from requests.exceptions import RequestException

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-6
"""

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'zh-CN,zh;q=0.9',
    'cookie': 'PHPSESSID=36c8n4lsbb8u63glevh1ksc9a1; webp_enabled=1; _ga=GA1.2.1167535880.1534758916; _gid=GA1.2.1330668796.1534758916; weilisessionid=aa3bf69b4f35c91ca4866315f1f300b1; wluuid=WLGEUST-02ADBA37-4B6C-DE33-2769-8697C4B575BB; wlsource=tc_pc_home; webp_enabled=0; _ga=GA1.3.1167535880.1534758916; _gid=GA1.3.1330668796.1534758916; _ba=BA0.2-20180820-51751-eyUyUL4rqUHUI1lh6uRM; qimo_seosource_e7dfc0b0-b3b6-11e7-b58e-df773034efe4=%E5%85%B6%E4%BB%96%E7%BD%91%E7%AB%99; qimo_seokeywords_e7dfc0b0-b3b6-11e7-b58e-df773034efe4=%E6%9C%AA%E7%9F%A5; accessId=e7dfc0b0-b3b6-11e7-b58e-df773034efe4; pageViewNum=1; bad_ide7dfc0b0-b3b6-11e7-b58e-df773034efe4=3c85f321-a45f-11e8-92ed-072415955da9; nice_ide7dfc0b0-b3b6-11e7-b58e-df773034efe4=3c85f322-a45f-11e8-92ed-072415955da9',
    'dnt': '1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}


# 获取imageID
def get_imageID(term, page):
    try:
        print('获取图片ID.....')
        url = 'https://stock.tuchong.com/api/free/search/?term=' + term + '&page=' + str(page)
        req = requests.get(url, headers=headers)
        if req.status_code == 200:
            json_imageid = req.json()
            return parse_imgID(json_imageid)
    except ConnectionError:
        return None


# 解析imageID里面的图片id
def parse_imgID(imageID):
    print('解析imageID')
    data = imageID.get('data')
    hits = data.get('hits')
    if hits:
        print('存在ID,解析')
        for item in hits:
            Id = item.get('imageId')
            get_ImageJPG(Id)
        return True


# 拼接图片ID获取图片url
def get_ImageJPG(id):
    if id:
        try:
            print('拼接url访问网页')
            url = 'https://stock.tuchong.com/free/image/?imageId=' + str(id)
            req = requests.get(url, headers=headers)
            if req.status_code == 200:
                return parse_imgURL(req.text)
        except ConnectionError:
            return None


# 解析html里面的图片url
def parse_imgURL(html):
    if html:
        print('解析HTML图片URL...')
        url = re.findall('<div.*?class="image-cover".*?<img.*?src="(.*?)">.*?</div>', html, re.S)
        # url = re.findall('<title>(.*?)</title>', html, re.S)
        for item in url:
            print("准备下载...", item)
            download_image(item)
    return None


def download_image(url):
    try:
        urls = 'https:' + url
        ir = requests.get(urls, headers=headers)
        if ir.status_code == 200:
            save_image(ir.content)
        return None
    except RequestException:
        return None


def save_image(content):
    file_path = '{0}/{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')
    if not os.path.exists(file_path):
        with open(file_path, 'wb') as f:
            f.write(content)
            f.close()
            print('下载成功----------------------')


def main():
    term = input('输入想要搜索的内容: ')
    for i in range(1, 7):
        get_imageID(term, i)


if __name__ == '__main__':
    main()



File: qsbk/qiushibaike.py

# !/usr/bin/python3
# -*- coding: utf-8 -*-

# 1. 导入线程池模块
# 线程池
import gevent.monkey
gevent.monkey.patch_all()
from gevent.pool import Pool
from queue import Queue
import requests
from lxml import etree

class QiushiSpider():

    def __init__(self, max_page):
        self.max_page = max_page
        # 2. 创建线程池，初始化线程数量
        self.pool = Pool(5)

        self.base_url = "http://www.qiushibaike.com/8hr/page/{}/"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
        }

        # 专门存放 url 容器
        self.url_queue = Queue()
        pass

    def get_url_list(self):
        '''
        获取 url 列表放入到 url 容器中
        :return:
        '''
        for page in range(1,self.max_page,1):
            url = self.base_url.format(page)
            self.url_queue.put(url)

    # 3. 实现执行任务
    def exec_task(self):
        # 1> 获取url
        url = self.url_queue.get()

        # 2> 发送请求获取 html
        response = requests.get(url,headers=self.headers)
        html = response.text

        # 3> 解析 html 提取数据
        eroot = etree.HTML(html)

        titles = eroot.xpath('//a[@class="recmd-content"]/text()')
        for title in titles:
            item = {}
            item["title"] = title

            # 4> 保存数据
            print(item)
        self.url_queue.task_done()



    # 4. 实现执行任务完成后的操作,必须至少有一个参数
    # result 任务执行的最终结果的返回值
    def exec_task_finished(self,result):
        print("result:",result)
        print("执行任务完成")
        self.pool.apply_async(self.exec_task,callback=self.exec_task_finished)


    def run(self):

        self.get_url_list()

        # 5. 让任务使用线程池中的线程执行并且设置执行后的回调操作
        # callback 表示执行完成后的回调
        for i in range(5):
            self.pool.apply_async(self.exec_task,callback=self.exec_task_finished)
        self.url_queue.join()
        pass

if __name__ == '__main__':
    max_page = input("请输入您需要多少页内容：")
    spider = QiushiSpider(int(max_page))
    spider.run()



File: 163youdao/163youdao.py

import time
from selenium import webdriver


login_url = "http://account.youdao.com/login?service=dict"

xpaths = {'usernameTxtBox': ".//*[@id='username']",
          'passwordTxtBox': ".//*[@id='password']",
          'submitButton': ".//*[@id='login']/div[2]/div/div[1]/form/p[4]/nobr/input",
          }


def login():
    mydriver = webdriver.Firefox()
    mydriver.get(login_url)
    mydriver.maximize_window()

    # Clear Username TextBox if already allowed "Remember Me"
    mydriver.find_element_by_xpath(xpaths['usernameTxtBox']).clear()

    username = input('Please type your user name:\n')

    # Write Username in Username TextBox
    mydriver.find_element_by_xpath(xpaths['usernameTxtBox']).send_keys(username)

    # Clear Password TextBox if already allowed "Remember Me"
    mydriver.find_element_by_xpath(xpaths['passwordTxtBox']).clear()

    password = input('Please type your password:\n')

    # Write Password in password TextBox
    mydriver.find_element_by_xpath(xpaths['passwordTxtBox']).send_keys(password)

    # Click Login button
    mydriver.find_element_by_xpath(xpaths['submitButton']).click()

    # add sleep
    print('登录成功')
    time.sleep(5)
          

if __name__ == '__main__':
    login()



File: Github/login.py

# -*- coding: utf-8 -*-
# @Author: CriseLYJ
# @Date:   2020-08-14 12:13:11

import re
import requests
from getpass import getpass


class GithubLogin(object):

    def __init__(self, email, password):
        # 初始化信息
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
            'Referer': 'https://github.com/',
            'Host': 'github.com'
        }

        self.session = requests.Session()
        self.login_url = 'https://github.com/login'
        self.post_url = 'https://github.com/session'
        self.email = email
        self.password = password

    def login_GitHub(self):
        # 登录入口
        post_data = {
            'commit': 'Sign in',
            'utf8': '✓',
            'authenticity_token': self.get_token(),
            'login': self.email,
            'password': self.password
        }
        resp = self.session.post(
            self.post_url, data=post_data, headers=self.headers)
        
        print('StatusCode:', resp.status_code)
        if resp.status_code != 200:
            print('Login Fail')
        match = re.search(r'"user-login" content="(.*?)"', resp.text)
        user_name = match.group(1)
        print('UserName:', user_name)



    # Get login token
    def get_token(self):

        response = self.session.get(self.login_url, headers=self.headers)

        if response.status_code != 200:
            print('Get token fail')
            return None
        match = re.search(
            r'name="authenticity_token" value="(.*?)"', response.text)
        if not match:
            print('Get Token Fail')
            return None
        return match.group(1)


if __name__ == '__main__':
    email = input('Account:')
    password = getpass('Password:')

    login = GithubLogin(email, password)
    login.login_GitHub()



File: facebook/facebook.py

from __future__ import print_function

import argparse
import requests
import pyquery


def login(session, email, password):
    """
    获取cookie
    """
    response = session.get('https://m.facebook.com')

    # 尝试登陆
    response = session.post('https://m.facebook.com/login.php', data={
        'email': email,
        'pass': password
    }, allow_redirects=False)

    if 'c_user' in response.cookies:
        # 说明登陆成功
        homepage_resp = session.get('https://m.facebook.com/home.php')

        dom = pyquery.PyQuery(homepage_resp.text.encode('utf8'))
        fb_dtsg = dom('input[name="fb_dtsg"]').val()

        return fb_dtsg, response.cookies['c_user'], response.cookies['xs']
    else:
        return False


if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='Login to Facebook')
    parser.add_argument('email', help='Email address')
    parser.add_argument('password', help='Login password')

    args = parser.parse_args()

    session = requests.session()
    session.headers.update({
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
    })

    fb_dtsg, user_id, xs = login(session, args.email, args.password)

    if user_id:
        print('{0}:{1}:{2}'.format(fb_dtsg, user_id, xs))
    else:
        print('Login Failed')



File: qqzone/qq_zone.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-7
"""

import time  # 用来延时
from selenium import webdriver

driver = webdriver.Chrome()  # 选择浏览器，此处我选择的Chrome
QQ_NUMBER = input('请输入你的QQ号')
PASSWORD = input('请输入你的QQ密码')

driver.get('http://i.qq.com/')
driver.switch_to.frame('login_frame')
driver.find_element_by_id('switcher_plogin').click()

driver.find_element_by_name('u').clear()
driver.find_element_by_name('u').send_keys(QQ_NUMBER)  # 此处输入你的QQ号
driver.find_element_by_name('p').clear()
driver.find_element_by_name('p').send_keys(PASSWORD)  # 此处输入你的QQ密码

driver.execute_script("document.getElementById('login_button').parentNode.hidefocus=false;")

driver.find_element_by_xpath('//*[@id="loginform"]/div[4]/a').click()
driver.find_element_by_id('login_button').click()

time.sleep(10)  # 因为我曾经是QQ会员，所以每次登陆时都会提醒我要不要再续费的弹窗...

driver.find_element_by_id('dialog_button_1').click()  # 这个地方是我把那个弹窗给点击了，配合上面的延时用的，延时是等待那个弹窗出现，然后此处点击取消

btns = driver.find_elements_by_css_selector('a.item.qz_like_btn_v3')  # 此处是CSS选择器
for btn in btns:
    btn.click()



File: tieba/tieba_spider.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-6
"""

"""
请求URL分析	https://tieba.baidu.com/f?kw=魔兽世界&ie=utf-8&pn=50
请求方式分析	GET
请求参数分析	pn每页50发生变化，其他参数固定不变
请求头分析	只需要添加User-Agent
"""

# 代码实现流程
# 1. 实现面向对象构建爬虫对象
# 2. 爬虫流程四步骤
# 2.1 获取url列表
# 2.2 发送请求获取响应
# 2.3 从响应中提取数据
# 2.4 保存数据

import requests


class TieBa_Spier():

    def __init__(self, max_pn, kw):
        # 初始化
        self.max_pn = max_pn
        self.kw = kw
        self.base_url = "https://tieba.baidu.com/f?kw={}&ie=utf-8&pn={}"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
        }

    def get_url_list(self):
        """获取url列表"""
        return [self.base_url.format(self.kw, pn) for pn in range(0, self.max_pn, 50)]

    def get_content(self, url):
        """发送请求获取响应内容"""
        response = requests.get(
            url=url,
            headers=self.headers
        )
        # print(response.text)
        return response.content

    def save_items(self, content, idx):
        """从响应内容中提取数据"""
        with open('{}.html'.format(idx), 'wb') as f:
            f.write(content)
        return None

    def run(self):
        """运行程序"""
        # 获取url_list
        url_list = self.get_url_list()

        for url in url_list:
            # 发送请求获取响应
            content = self.get_content(url)

            # 保存数据
            items = self.save_items(content, url_list.index(url) + 1)

            # 测试
            # print(items)


if __name__ == '__main__':
    spider = TieBa_Spier(200, "英雄联盟")
    spider.run()



File: NeteaseCloudMusicDownload/api.py

# -*- coding: utf-8 -*-
# @Author: CriseLYJ
# @Date:   2020-08-14 13:48:23

import requests
import math
import random
from Crypto.Cipher import AES
import base64
import codecs
import os


class decrypt_music(object):
    def __init__(self, d):
        self.d = d
        self.e = '010001'
        self.f = "00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5a" \
                 "a76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46be" \
                 "e255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7"
        self.g = '0CoJUm6Qyw8W8jud'
        self.random_text = self.get_random_str()

    def get_random_str(self):
        str = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
        res = ''
        for x in range(16):
            index = math.floor(random.random() * len(str))
            res += str[index]
        return res

    def aes_encrypt(self, text, key):
        iv = b'0102030405060708'
        pad = 16 - len(text.encode()) % 16
        text = text + pad * chr(pad)
        # fix: https://github.com/Kr1s77/awesome-python-login-model/issues/100#issuecomment-673897848
        # error: TypeError: Object type <class 'str'> cannot be passed to C code
        encryptor = AES.new(key.encode(), AES.MODE_CBC, iv)
        msg = base64.b64encode(encryptor.encrypt(text.encode()))
        return msg

    def rsa_encrypt(self, value, text, modulus):
        '''进行rsa加密'''
        text = text[::-1]
        rs = int(codecs.encode(text.encode('utf-8'), 'hex_codec'), 16) ** int(value, 16) % int(modulus, 16)
        return format(rs, 'x').zfill(256)

    def get_data(self):
        params = self.aes_encrypt(self.d, self.g)
        params = self.aes_encrypt(params.decode('utf-8'), self.random_text)
        enc_sec_key = self.rsa_encrypt(self.e, self.random_text, self.f)
        return {
            'params': params,
            'encSecKey': enc_sec_key
        }


class Spider(object):
    def __init__(self):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0',
            'Cookie': '_iuqxldmzr_=32; _ntes_nnid=8d4ef0883a3bcc9d3a2889b0bf36766a,1533782432391; _ntes_nuid=8d4ef0883a3bcc9d3a2889b0bf36766a; __utmc=94650624; WM_TID=GzmBlbRkRGQXeQiYuDVCfoEatU6VSsKC; playerid=19729878; __utma=94650624.1180067615.1533782433.1533816989.1533822858.9; __utmz=94650624.1533822858.9.7.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/; WM_NI=S5gViyNVs14K%2BZoVerGK69gLlmtnH5NqzyHcCUY%2BiWm2ZaHATeI1gfsEnK%2BQ1jyP%2FROzbzDV0AyJHR4YQfBetXSRipyrYCFn%2BNdA%2FA8Mv80riS3cuMVJi%2BAFgCpXTiHBNHE%3D; WM_NIKE=9ca17ae2e6ffcda170e2e6ee84b674afedfbd3cd7d98b8e1d0f554f888a4abc76990b184badc4f89e7af8ece2af0fea7c3b92a91eba9b7ec738e8abdd2b741e986a1b7e87a8595fadae648b0b3bc8fcb3f8eafb69acb69818b97ccec5dafee9682cb4b98bb87d2e66eb19ba2acaa5bf3b6b7b1ae5a8da6ae9bc75ef49fb7abcb5af8879f87c16fb8889db3ec7cbbae97a4c566e992aca2ae4bfc93bad9b37aab8dfd84f8479696a7ccc44ea59dc0b9d7638c9e82a9c837e2a3; JSESSIONID-WYYY=sHwCKYJYxz6ODfURChA471BMF%5CSVf3%5CTc8Qcy9h9Whj6CfMxw4YWTMV7CIx5g6rqW8OBv04YGHwwq%2B%5CD1N61qknTP%2Fym%2BHJZ1ylSH1EabbQASc9ywIT8YvOr%2FpMgvmm1cbr2%2Bd6ssMYXuTlpOIrKqp%5C%2FM611EhmfAfU47%5CSQWAs%2BYzgY%3A1533828139236'

        }

    def __get_songs(self, name):
        d = '{"hlpretag":"<span class=\\"s-fc7\\">","hlposttag":"</span>","s":"%s","type":"1","offset":"0","total":"true","limit":"30","csrf_token":""}' % name
        wyy = decrypt_music(d)
        data = wyy.get_data()
        url = 'https://music.163.com/weapi/cloudsearch/get/web?csrf_token='
        response = requests.post(url, data=data, headers=self.headers).json()
        return response['result']

    def __get_mp3(self, id):
        d = '{"ids":"[%s]","br":320000,"csrf_token":""}' % id
        wyy = decrypt_music(d)
        data = wyy.get_data()
        url = 'https://music.163.com/weapi/song/enhance/player/url?csrf_token='
        response = requests.post(url, data=data, headers=self.headers).json()
        print(response)
        return response['data'][0]['url']

    def __download_mp3(self, url, filename):
        abspath = os.path.abspath('.')
        os.chdir(abspath)
        response = requests.get(url, headers=self.headers).content
        path = os.path.join(abspath, filename)
        with open(filename + '.mp3', 'wb') as f:
            f.write(response)
            print('下载完毕,可以在%s   路径下查看' % path + '.mp3')

    def __print_info(self, songs):
        """打印歌曲需要下载的歌曲信息"""
        songs_list = []
        for num, song in enumerate(songs):
            print(num, '歌曲名字：', song['name'], '作者：', song['ar'][0]['name'])
            songs_list.append((song['name'], song['id']))
        return songs_list

    def run(self):
        while True:
            name = input('请输入你需要下载的歌曲：')
            songs = self.__get_songs(name)
            if songs['songCount'] == 0:
                print('没有搜到此歌曲，请换个关键字')
            else:
                songs = self.__print_info(songs['songs'])
                num = input('请输入需要下载的歌曲，输入左边对应数字即可')
                url = self.__get_mp3(songs[int(num)][1])
                if not url:
                    print('歌曲需要收费，下载失败')
                else:
                    filename = songs[int(num)][0]
                    self.__download_mp3(url, filename)
                flag = input('如需继续可以按任意键进行搜歌，否则按0结束程序')
                if flag == '0':
                    break
        print('程序结束！')


if __name__ == '__main__':
    spider = Spider()
    spider.run()



File: 126email/126email.py

import time
from selenium import webdriver
from getpass import getpass

def login():
    acount_num = input('请输入账号:')
    passwd_str = getpass('请输入密码:')
    driver = webdriver.Chrome()
    url = 'http://mail.126.com/'
    driver.get(url)
    time.sleep(30)

    elem = driver.find_element_by_css_selector("iframe[id^='x-URS-iframe']")
    # # 126登陆框是使用iframe进行嵌套的，所以需要先切换到该iframe
    driver.switch_to.frame(elem)

    acount = driver.find_element_by_name('email')
    acount.clear()
    acount.send_keys(acount_num)

    passwd = driver.find_element_by_name('password')
    passwd.clear()
    passwd.send_keys(passwd_str)

    time.sleep(3)
    click_button = driver.find_element_by_id('dologin')
    click_button.click()
    time.sleep(5)
    cur_cookies = driver.get_cookies()[0]
    return cur_cookies


if __name__ == '__main__':
    login()



File: liepin/liepin_login.py

# -*- coding: utf-8 -*-
'''
Required
- requests 
- bs4
'''
# 输入密码不可见模块导入
import getpass
import hashlib
import requests
from bs4 import BeautifulSoup


class Leipin(object):
    def __init__(self, username, password):
        self.username = username
        self.password = password
        self.headers = {
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Accept-Encoding': 'gzip, deflate, sdch',
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'Connection': 'keep-alive',
            'Content-Type': 'application/x-www-form-urlencoded',
            'Host': 'passport.liepin.com',
            'User-Agent': "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
            'X-Requested-With': 'XMLHttpRequest',
            'Upgrade-Insecure-Requests': '1',
        }
        self.proxies = {
            'HTTP': 'http://120.198.231.88:80',
            # 'HTTP':'http://222.174.71.46:9999',
        }  # 测试所用代理
        self.session = requests.session()
        self.accountUrl = 'https://passport.liepin.com/h/account'
        self.loginUrl = 'https://passport.liepin.com/h/login.json'
        self.Dir = 'E:\\python\\authcode.jpg'  # authcode folder 根据自己情况选择文件夹

    def _md5(self):
        md5 = hashlib.md5()
        md5.update(str.encode(password))
        psw = md5.hexdigest()
        return psw

    def _getAuthcode(self):
        r = self.session.get(self.accountUrl, headers=self.headers, timeout=10, proxies=self.proxies)
        page = BeautifulSoup(r.text)
        selector = page.find_all('div', class_="ui-tab-toggle hide")[0]
        imageUrl = selector.select('div > img')[0]['src']
        authcodeUrl = 'https://passport.liepin.com{}'.format(imageUrl)
        response = self.session.get(authcodeUrl)
        if response.status_code == 200:
            with open(self.Dir, 'wb') as f:
                f.write(response.content)
        authcode = input('plz input authcode:')
        return authcode

    def login(self):
        payload = {
            'user_login': self.username,
            'isMd5': 1,
            'user_pwd': self._md5(),
            'user_kind': 2,  # 根据你是否为正式会员而定，根据自身情况可能需要修改
            'verifycode': self._getAuthcode(),
            'url': '',
        }
        del self.headers['Upgrade-Insecure-Requests']
        self.headers['Origin'] = 'https://passport.liepin.com'
        self.headers['Referer'] = 'https://passport.liepin.com/h/account'
        response = self.session.post(self.loginUrl, headers=self.headers, data=payload, timeout=10,
                                     proxies=self.proxies, allow_redirects=False)
        return response.status_code, response.text


if __name__ == '__main__':
    userName = input("请输入你的用户名:")
    password = getpass.getpass("password:")
    lp = Leipin(userName, password)
    print(lp.login())



File: liepin/liepinSpecialCom/run_liepinspecialcom.py

# !/usr/bin/env python
# -*- coding: utf-8 -*-

# 获取settings.py模块的设置
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from liepinSpecialCom.spiders.lpspecialcom import LiepinSpdier

settings = get_project_settings()
process = CrawlerProcess(settings=settings)

# 可以添加多个spider类
process.crawl(LiepinSpdier)

# 启动爬虫，会阻塞，直到爬取完成
process.start()


File: liepin/liepinSpecialCom/liepinSpecialCom/__init__.py




File: liepin/liepinSpecialCom/liepinSpecialCom/middlewares.py

# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
from scrapy import signals
import scrapy
import random
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware


class LiepinspecialcomSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class LiepinspecialcomDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class MyUserAgentMiddleware(UserAgentMiddleware):
    '''
    设置User-Agent
    '''

    def __init__(self, user_agent):
        self.user_agent = user_agent

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            user_agent=crawler.settings.get('USER_AGENTS')
        )

    def process_request(self, request, spider):
        agent = random.choice(self.user_agent)
        request.headers['User-Agent'] = agent



File: liepin/liepinSpecialCom/liepinSpecialCom/settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for liepinSpecialCom project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'liepinSpecialCom'

MYSQL_HOST = 'localhost'
MYSQL_DBNAME = 'day0123'
MYSQL_USER = 'root'
MYSQL_PASSWD = '123'

SPIDER_MODULES = ['liepinSpecialCom.spiders']
NEWSPIDER_MODULE = 'liepinSpecialCom.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'liepinSpecialCom (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    'Connection': 'keep-alive',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9'
}

USER_AGENTS = [
    "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
    "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
    "Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
    "Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0"
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"
    "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
    "Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
    "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
    "Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
    'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
    'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
    'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)'
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
]

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'liepinSpecialCom.middlewares.LiepinspecialcomSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   # 'liepinSpecialCom.middlewares.LiepinspecialcomDownloaderMiddleware': 543,
    'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
    'liepinSpecialCom.middlewares.MyUserAgentMiddleware': 400,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'liepinSpecialCom.pipelines.LiepinspecialcomPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'



File: liepin/liepinSpecialCom/liepinSpecialCom/items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class LiepinspecialcomItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    as_of_date = scrapy.Field()
    ticker = scrapy.Field()
    company_name = scrapy.Field()
    # stage = scrapy.Field()
    size = scrapy.Field()
    city = scrapy.Field()
    industry = scrapy.Field()
    # comp_clearfix = scrapy.Field()
    # job_count = scrapy.Field()
    # rate_num = scrapy.Field()
    # registered_capital = scrapy.Field()



File: liepin/liepinSpecialCom/liepinSpecialCom/pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

from twisted.enterprise import adbapi
import pymysql
import pymysql.cursors

# class LiepinspdPipeline(object):
    # def __init__(self, dbpool):
    #     self.dbpool = dbpool
    #
    # @classmethod
    # def from_settings(cls, settings):  # 函数名固定，会被scrapy调用，直接可用settings的值
    #     """
    #     数据库建立连接
    #     :param settings: 配置参数
    #     :return: 实例化参数
    #     """
    #
    #     adbparams = dict(
    #         host=settings['MYSQL_HOST'],
    #         db=settings['MYSQL_DBNAME'],
    #         user=settings['MYSQL_USER'],
    #         password=settings['MYSQL_PASSWORD'],
    #         port = settings['MYSQL_PORT'],
    #         cursorclass=pymysql.cursors.DictCursor  # 指定cursor类型
    #     )
    #     # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
    #     dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
    #     # 返回实例化参数
    #     return cls(dbpool)
    #
    # def process_item(self, item, spider):
    #     """
    #     使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
    #     """
    #     query = self.dbpool.runInteraction(self.do_insert, item)  # 指定操作方法和操作数据
    #     # 添加异常处理
    #     query.addCallback(self.handle_error)  # 处理异常
    #
    # def do_insert(self, cursor, item):
    #     # 对数据库进行插入操作，并不需要commit，twisted会自动commit
    #
    #     insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industy,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
    #     cursor.execute(insert_sql,
    #                    (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
    #                     str(item['size']), str(item['city']), str(item['industy']), str(item['comp_clearfix']),
    #                     int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),item['spider_time'],item['origin_site'],))
    # def handle_error(self, failure):
    #     if failure:
    #         # 打印错误信息
    #         print(failure)


import pymysql


class LiepinspecialcomPipeline(object):
    """
    同步操作
    """

    def __init__(self):
        # 建立连接
        self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data')  # 有中文要存入数据库的话要加charset='utf8'
        # 创建游标
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        # sql语句
        insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,`size`,city,industry) VALUES(%s,%s,%s,%s,%s,%s)"
        # 执行插入数据到数据库操作
        self.cursor.execute(insert_sql,
                            (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['size']),
                                str(item['city']),str(item['industry'])))
        # 提交，不进行提交无法保存到数据库
        self.conn.commit()

    def close_spider(self, spider):
        # 关闭游标和连接
        self.cursor.close()
        self.conn.close()



# class LiepinspecialcomPipeline(object):
#     def __init__(self, dbpool):
#         self.dbpool = dbpool
#
#     @classmethod
#     def from_settings(cls, settings):  # 函数名固定，会被scrapy调用，直接可用settings的值
#         """
#         数据库建立连接
#         :param settings: 配置参数
#         :return: 实例化参数
#         """
#
#         adbparams = dict(
#             host=settings['MYSQL_HOST'],
#             db=settings['MYSQL_DBNAME'],
#             user=settings['MYSQL_USER'],
#             password=settings['MYSQL_PASSWORD'],
#             cursorclass=pymysql.cursors.DictCursor  # 指定cursor类型
#         )
#         # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
#         dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
#         # 返回实例化参数
#         return cls(dbpool)
#
#     def process_item(self, item, spider):
#         """
#         使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
#         """
#         query = self.dbpool.runInteraction(self.do_insert, item)  # 指定操作方法和操作数据
#         # 添加异常处理
#         query.addCallback(self.handle_error)  # 处理异常
#
#     def do_insert(self, cursor, item):
#         # 对数据库进行插入操作，并不需要commit，twisted会自动commit
#         insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,`size`,city,industry) VALUES(%s,%s,%s,%s,%s,%s)"
#         cursor.execute(insert_sql, (
#         item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['size']), str(item['city']),
#         str(item['industry'])))
#
#     def handle_error(self, failure):
#         if failure:
#             # 打印错误信息
#             print(failure)



File: liepin/liepinSpecialCom/liepinSpecialCom/spiders/__init__.py

# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.



File: liepin/liepinSpecialCom/liepinSpecialCom/spiders/lpspecialcom.py

import scrapy
import re
from datetime import datetime
import pandas as pd
import time

from liepinSpd.items import LiepinspdItem


class LiepinSpdier(scrapy.Spider):
    name = 'liepin'
    start_urls = ['https://www.liepin.com/job/1917579081.shtml?d_sfrom=search_comp&d_ckId=d112af69ab58e7da8305520f55b31904&d_curPage=0&d_pageSize=15&d_headId=d112af69ab58e7da8305520f55b31904&d_posi=0',
                  'https://www.liepin.com/job/1917549017.shtml?d_sfrom=search_comp&d_ckId=1bef90aa98c2e8da734552c320527ac0&d_curPage=0&d_pageSize=15&d_headId=1bef90aa98c2e8da734552c320527ac0&d_posi=0',
                  'https://www.liepin.com/job/1917543155.shtml',
                  'https://www.liepin.com/job/1917491571.shtml?d_sfrom=search_comp&d_ckId=5874ecde43eb4bd20e75fecb2709bf85&d_curPage=0&d_pageSize=15&d_headId=5874ecde43eb4bd20e75fecb2709bf85&d_posi=0',
                  'https://www.liepin.com/job/1917505785.shtml?d_sfrom=search_comp&d_ckId=fe82f0f79cda01b1dd4c140ced26087c&d_curPage=0&d_pageSize=15&d_headId=fe82f0f79cda01b1dd4c140ced26087c&d_posi=0',
                  'https://www.liepin.com/job/1916439263.shtml?d_sfrom=search_comp&d_ckId=d3f4428da37a0cd17a6235cb4a027f1e&d_curPage=0&d_pageSize=15&d_headId=d3f4428da37a0cd17a6235cb4a027f1e&d_posi=0',
                  'https://www.liepin.com/job/1911157736.shtml?d_sfrom=search_comp&d_ckId=2cf44398e8273003087d5148e113ef8f&d_curPage=0&d_pageSize=15&d_headId=2cf44398e8273003087d5148e113ef8f&d_posi=0',
                  'https://www.liepin.com/job/1917470663.shtml?d_sfrom=search_comp&d_ckId=9087e4fc55d61d200606fb906999f728&d_curPage=0&d_pageSize=15&d_headId=9087e4fc55d61d200606fb906999f728&d_posi=0',
                  'https://www.liepin.com/job/1917533673.shtml?d_sfrom=search_comp&d_ckId=98408645fba7219d4d7f17f2714c96f0&d_curPage=0&d_pageSize=15&d_headId=98408645fba7219d4d7f17f2714c96f0&d_posi=0',
                  'https://www.liepin.com/job/1917306593.shtml?d_sfrom=search_comp&d_ckId=85f632646e2b1ad7c06f436e25fd674d&d_curPage=0&d_pageSize=15&d_headId=85f632646e2b1ad7c06f436e25fd674d&d_posi=0',
                  'https://www.liepin.com/job/199929552.shtml'
                    ]

    # 公司主要基本信息
    def parse(self, response):
        text = response.text
        # print(text)
        # 抓取公司基本信息
        # try:
        company_name = response.xpath('//div[@class="about-position"]//a/text()')[0].extract()
        # print(company_name)
        # comp_sum_tag = response.xpath('//div[@class="comp-summary-tag"]/a/text()').extract()
        # 好几个
        # stage = comp_sum_tag[0]
        # print(stage)
        size = re.search(r'公司规模：(.*?)人',text).group(1)
        # print(size)
        city = re.search(r'公司地址：(.*?)<',text).group(1)
        # print(city)
        industry = re.search(r'行业.*?>(.*?)<',text).group(1)
        # print(industy)
        # 公司标签,list
        # comp_clearfix = str(response.xpath('//ul[@class="comp-tag-list clearfix"]//span/text()').extract())
        # print(comp_clearfix)
        # 简历处理率   *%转化为float
        # rate_num = response.xpath('//p[@class="rate-num"]//span/text()')[0].extract()
        # rate_num = int(rate_num) / 100
        # print(rate_num)

        # job_count = int(re.search(r'<small data-selector="total">. 共([0-9]+) 个', text).group(1))
        # print(job_count)
        # 注册资本(万元)
        # registered_capital = float(re.search(r'<li>注册资本：(.*?)万元人民币</li>', text).group(1))
        # print(registered_capital)

        as_of_date = datetime.now()  # 最后确认一下格式是否正确

        item = LiepinspdItem()
        # 匹配股票代码,判断如果股票简称全部在公司名内,则匹配股票代码
        data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
        try:
            for i in range(len(data)):
                n = 0
                for j in data.loc[i, '股票简称']:
                    if j in company_name:
                        n += 1
                if n >= len(data.loc[i, '股票简称'])-1:
                    item['ticker'] = data.loc[i, '股票代码']
                    print(n, item['ticker'], company_name)
        except BaseException as e:
            item['ticker'] ='None'
            print('ticker匹配错误')

        item['as_of_date'] = as_of_date
        item['company_name'] = company_name
        # item['stage'] = stage
        item['size'] = size
        item['city'] = city
        item['industry'] = industry
        # item['comp_clearfix'] = comp_clearfix
        # item['rate_num'] = rate_num
        # item['job_count'] = job_count
        # item['registered_capital'] = registered_capital
        # time.sleep(2)

        yield item
        # except BaseException as e:
        #     print('error and pass')


File: liepin/liepinSpecialComJob/run_liepinspecialjob.py

# !/usr/bin/env python
# -*- coding: utf-8 -*-

# 获取settings.py模块的设置
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from liepinSpecialComJob.spiders.lpspecialcomjob import LiepinSpdier

settings = get_project_settings()
process = CrawlerProcess(settings=settings)

# 可以添加多个spider类
process.crawl(LiepinSpdier)

# 启动爬虫，会阻塞，直到爬取完成
process.start()


File: liepin/liepinSpecialComJob/liepinSpecialComJob/__init__.py




File: liepin/liepinSpecialComJob/liepinSpecialComJob/middlewares.py

# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
import random
import time

from scrapy import signals
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware

class LiepinspecialcomjobSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class LiepinspecialcomjobDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class MyUserAgentMiddleware(UserAgentMiddleware):
    '''
    设置User-Agent
    '''

    def __init__(self, user_agent):
        self.user_agent = user_agent

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            user_agent=crawler.settings.get('USER_AGENTS')
        )

    def process_request(self, request, spider):
        agent = random.choice(self.user_agent)
        request.headers['User-Agent'] = agent
        print(agent)


class ProxyMiddleware(object):
    """docstring for ProxyMiddleWare"""

    def process_request(self, request, spider):
        '''对request对象加上proxy'''
        proxy = self.get_random_proxy()
        print("this is request ip:" + proxy)
        request.meta['proxy'] = proxy

    def process_response(self, request, response, spider):
        '''对返回的response处理'''
        # 如果返回的response状态不是200，重新生成当前request对象
        if response.status != 200:
            proxy = self.get_random_proxy()
            print("this is response ip:" + proxy)
            # 对当前reque加上代理
            request.meta['proxy'] = proxy
            return request
        return response

    def get_random_proxy(self):
        '''随机从文件中读取proxy'''

        while 1:
            with open('G:\workspace\common\proxies.txt', 'r') as f:
                proxies = f.readlines()
            if proxies:
                break
            else:
                time.sleep(1)
        proxy = random.choice(proxies).strip()
        return proxy



File: liepin/liepinSpecialComJob/liepinSpecialComJob/settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for liepinSpecialComJob project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'liepinSpecialComJob'

SPIDER_MODULES = ['liepinSpecialComJob.spiders']
NEWSPIDER_MODULE = 'liepinSpecialComJob.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'liepinSpecialComJob (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
MYSQL_HOST = 'localhost'
MYSQL_DBNAME = 'day0123'
MYSQL_USER = 'root'
MYSQL_PASSWD = '123'

DEFAULT_REQUEST_HEADERS = {
    'Connection': 'keep-alive',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9'
}

USER_AGENTS = [
    "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
    "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
    "Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
    "Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
    "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
    "Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
    "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
    "Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
    'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
    'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
    'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
]

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'liepinSpecialComJob.middlewares.LiepinspecialcomjobSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
    # 'liepinSpd2.middlewares.Liepinspd2DownloaderMiddleware': 543,
    'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
    'liepinSpd2.middlewares.MyUserAgentMiddleware': 400,
    # 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': None,
    # 'liepinSpd2.middlewares.ProxyMiddleware': 750,
    # 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': None,
}

ITEM_PIPELINES = {
   'liepinSpecialComJob.pipelines.LiepinspecialcomjobPipeline': 300,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'



File: liepin/liepinSpecialComJob/liepinSpecialComJob/items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class LiepinspecialcomjobItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    as_of_date = scrapy.Field()
    ticker = scrapy.Field()
    company_name = scrapy.Field()
    job_name = scrapy.Field()
    salary = scrapy.Field()
    city = scrapy.Field()
    work_year = scrapy.Field()
    pub_time = scrapy.Field()
    education = scrapy.Field()
    origin_site = scrapy.Field()



File: liepin/liepinSpecialComJob/liepinSpecialComJob/pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

from twisted.enterprise import adbapi
import pymysql
import pymysql.cursors
import time


# class Liepinspd2Pipeline(object):
#     def __init__(self, dbpool):
#         self.dbpool = dbpool
#
#     @classmethod
#     def from_settings(cls, settings):  # 函数名固定，会被scrapy调用，直接可用settings的值
#         """
#         数据库建立连接
#         :param settings: 配置参数
#         :return: 实例化参数
#         """
#
#         adbparams = dict(
#             host=settings['MYSQL_HOST'],
#             db=settings['MYSQL_DBNAME'],
#             user=settings['MYSQL_USER'],
#             password=settings['MYSQL_PASSWORD'],
#             cursorclass=pymysql.cursors.DictCursor  # 指定cursor类型
#         )
#         # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
#         dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
#         # 返回实例化参数
#         return cls(dbpool)
#
#     def process_item(self, item, spider):
#         """
#         使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
#         """
#         query = self.dbpool.runInteraction(self.do_insert, item)  # 指定操作方法和操作数据
#         # 添加异常处理
#         query.addCallback(self.handle_error)  # 处理异常
#
#     def do_insert(self, cursor, item):
#         # 对数据库进行插入操作，并不需要commit，twisted会自动commit
#         insert_sql = "insert into liepin_job(as_of_date,ticker,company_name,job_name,salary,city,education,work_year,pub_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
#         cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),
#                                     str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['origin_site'])))
#
#     def handle_error(self, failure):
#         if failure:
#             # 打印错误信息
#             print(failure)

class LiepinspecialcomjobPipeline(object):
    """
    同步操作
    """

    def __init__(self):
        # 建立连接
        self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data')  # 有中文要存入数据库的话要加charset='utf8'
        # 创建游标
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        # sql语句
        insert_sql = 'insert into job_info(as_of_date,ticker,company_name,job_name,salary,city,education,work_year,pub_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
        # 执行插入数据到数据库操作
        self.cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),
                                    str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['origin_site'])))
        # 提交，不进行提交无法保存到数据库
        self.conn.commit()

    def close_spider(self, spider):
        # 关闭游标和连接
        self.cursor.close()
        self.conn.close()







#
#
# class LiepinspecialcomjobPipeline(object):
#     def __init__(self, dbpool):
#         self.dbpool = dbpool
#
#     @classmethod
#     def from_settings(cls, settings):  # 函数名固定，会被scrapy调用，直接可用settings的值
#         """
#         数据库建立连接
#         :param settings: 配置参数
#         :return: 实例化参数
#         """
#
#         adbparams = dict(
#             host=settings['MYSQL_HOST'],
#             db=settings['MYSQL_DBNAME'],
#             user=settings['MYSQL_USER'],
#             password=settings['MYSQL_PASSWORD'],
#             cursorclass=pymysql.cursors.DictCursor  # 指定cursor类型
#         )
#         # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
#         dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
#         # 返回实例化参数
#         return cls(dbpool)
#
#     def process_item(self, item, spider):
#         """
#         使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
#         """
#         query = self.dbpool.runInteraction(self.do_insert, item)  # 指定操作方法和操作数据
#         # 添加异常处理
#         query.addCallback(self.handle_error)  # 处理异常
#
#     def do_insert(self, cursor, item):
#         # 对数据库进行插入操作，并不需要commit，twisted会自动commit
#         insert_sql = "insert into liepin_job(as_of_date,ticker,company_name,job_name,salary,city,education,work_year,pub_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
#         cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),
#                                     str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['origin_site'])))
#
#     def handle_error(self, failure):
#         if failure:
#             # 打印错误信息
#             print(failure)


File: liepin/liepinSpecialComJob/liepinSpecialComJob/spiders/__init__.py

# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.



File: liepin/liepinSpecialComJob/liepinSpecialComJob/spiders/lpspecialcomjob.py

import json

import scrapy
import re
from datetime import datetime
import pandas as pd
import time
from common.util import get_13_time
t = get_13_time()
from liepinSpecialComJob.items import LiepinspecialcomjobItem


class LiepinSpdier(scrapy.Spider):
    name = 'liepin'
    start_urls = ['https://vip.liepin.com/883905/1405577359643.shtml',
                  'https://vip.liepin.com/8161070/joblist.shtml',
                  # 'http://maker.haier.net/custompage/socialchannel/index.html?platformcode=lp',
                  'https://vip.liepin.com/7855333/joblist.shtml',
                  'https://vip.liepin.com/8090130/1409730340536.shtml',
                  'https://vip.liepin.com/8399212/joblist.shtml',
                  'https://vip.liepin.com/1198424/joblist2.shtml',
                  'https://vip.liepin.com/8787971/joblist.shtml',
                  'https://vip.liepin.com/8796178/joblist2.shtml',
                  'https://vip.liepin.com/8091337/1426475303042.shtml',
                  'https://vip.liepin.com/7904788/job.shtml',
                    ]

    def parse(self, response):
        text = response.text
        company_name = re.search(r'<title>(.*?) - 猎聘网招聘官网',text).group(1)
        companyId=re.search(r'CONFIG={"companyId":"([0-9]+)"}',text).group(1)
        next_meta = response.meta
        data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
        try:
            for i in range(len(data)):
                n = 0
                for j in data.loc[i, '股票简称']:
                    if j in company_name:
                        n += 1
                if n == len(data.loc[i, '股票简称']):
                    next_meta['ticker'] = data.loc[i, '股票代码']
                    print(n, next_meta['ticker'], company_name)
        except BaseException as e:
            next_meta['ticker'] ='None'
            print('ticker匹配错误')
        next_meta['company_name'] = company_name
        next_meta['companyId'] = companyId
        url='https://www.liepin.com/ajaxproxy.html'
        # headers={
        #     'Referer':'https://vip.liepin.com/8091337/1426475303042.shtml'
        # }
        yield scrapy.Request(url, callback=self.parse_list, meta=next_meta,dont_filter=True)

    def parse_list(self,response):
        next_meta = response.meta
        companyId = next_meta['companyId'].strip()
        # print(companyId,response.text)
        n=0
        while n<95:
            # try:
            t = get_13_time()
            # 'https://www.liepin.com/company/sojob.json?pageSize=15&curPage=0&ecompIds=8091337&dq=&publishTime=&keywords=&_=1550383073951'
            url=f'https://www.liepin.com/company/sojob.json?pageSize=15&curPage={n}&ecompIds={companyId}&dq=&publishTime=&keywords=&_={t}'
            n+=1
            headers={
                'referer':'https://www.liepin.com/ajaxproxy.html'
            }
            cookies={
                '__uuid': '1550017147980.22',
                '_uuid': 'E4361B46FFA8441973EC46E6488BD983',
                'is_lp_user': 'true',
                'need_bind_tel': 'false',
                'new_user': 'false',
                'c_flag': 'f57e19ed294147b87179e4e6132477f5',
                'imClientId': '45e417dd37f82ac674cdcbb355984626',
                'imId': '45e417dd37f82ac6a36687782a0c1c67',
                'imClientId_0': '45e417dd37f82ac674cdcbb355984626',
                'imId_0': '45e417dd37f82ac6a36687782a0c1c67',
                'gr_user_id': '374534ce-aa54-4880-88ca-7a7bb7adf340',
                'bad1b2d9162fab1f80dde1897f7a2972_gr_last_sent_cs1': '463d81f04fd219c61a667e00ad0d9493',
                'grwng_uid': 'f3fda8f8-0c2e-4f29-8507-f42f7a9671ec',
                'fe_work_exp_add': 'true',
                'ADHOC_MEMBERSHIP_CLIENT_ID1.0': 'fa804ff0-2a02-3f31-8dcb-8e13b527dfcb',
                'bad1b2d9162fab1f80dde1897f7a2972_gr_cs1': '463d81f04fd219c61a667e00ad0d9493',
                '__tlog': '1550383052778.97%7C00000000%7C00000000%7C00000000%7C00000000',
                '_mscid': '00000000',
                'Hm_lvt_a2647413544f5a04f00da7eee0d5e200': '1550233873,1550279247,1550281552,1550383053',
                'abtest': '0',
                '_fecdn_': '0',
                '__session_seq': '2',
                '__uv_seq': '2',
                'Hm_lpvt_a2647413544f5a04f00da7eee0d5e200': '1550383074'
            }
            next_meta['ticker'] = next_meta['ticker']
            print(next_meta['ticker'])
            next_meta['company_name'] = next_meta['company_name']
            print(next_meta['company_name'])
            yield scrapy.Request(url, callback=self.parse_job,meta=next_meta,headers=headers,cookies=cookies)
            # except BaseException as e:
            #     print('已完成最后一页')
            #     break

    def parse_job(self,response):
        meta = response.meta
        item = LiepinspecialcomjobItem()
        text = response.text
        print('****************************************')
        json_data = json.loads(text)
        as_of_date = datetime.now()
        job_infos=json_data['list']
        for job_info in job_infos:
            origin_site=job_info['url']
            job_name=job_info['title']
            salary=job_info['salary']
            city=job_info['city']
            education=job_info['eduLevel']
            work_year=job_info['workYear']
            pub_time=job_info['time']
            function=job_info['dept']

            item['ticker'] = meta['ticker'].strip()
            item['company_name'] = meta['company_name'].strip()
            item['job_name']=job_name
            item['salary']=salary
            item['city']=city
            item['education']=education
            item['work_year']=work_year
            item['pub_time']=pub_time
            item['as_of_date']=as_of_date
            item['function']=function
            item['origin_site']=origin_site

            yield item



        #暂不深挖
        # for url in origin_sites:
        #     yield scrapy.Request(url, callback=self.parse_job)
















File: liepin/liepinSpd/run_liepin1.py

# !/usr/bin/env python
# -*- coding: utf-8 -*-

# 获取settings.py模块的设置
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from liepinSpd.spiders.lpspider import LiepinSpdier

settings = get_project_settings()
process = CrawlerProcess(settings=settings)

# 可以添加多个spider类
process.crawl(LiepinSpdier)

# 启动爬虫，会阻塞，直到爬取完成
process.start()


File: liepin/liepinSpd/liepinSpd/__init__.py




File: liepin/liepinSpd/liepinSpd/middlewares.py

# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
import scrapy
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
import random

class LiepinspdSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class LiepinspdDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class MyUserAgentMiddleware(UserAgentMiddleware):
    '''
    设置User-Agent
    '''

    def __init__(self, user_agent):
        self.user_agent = user_agent

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            user_agent=crawler.settings.get('USER_AGENTS')
        )

    def process_request(self, request, spider):
        agent = random.choice(self.user_agent)
        request.headers['User-Agent'] = agent


File: liepin/liepinSpd/liepinSpd/dbhelper.py

import pymysql
from scrapy.utils.project import get_project_settings#引入settings配置

class DBHelper():

    def __init__(self):
        self.settings=get_project_settings()#获取settings配置数据

        self.host=self.settings['MYSQL_HOST']
        self.port=self.settings['MYSQL_PORT']
        self.user=self.settings['MYSQL_USER']
        self.passwd=self.settings['MYSQL_PASSWD']
        self.db=self.settings['MYSQL_DBNAME']
    #连接mysql
    def connectMysql(self):
        conn=pymysql.connect(host=self.host,
                             port=self.port,
                             user=self.user,
                             passwd=self.passwd,
                             charset='utf8')
        return conn
    #连接数据库
    def connectDatabase(self):
        conn=pymysql.connect(host=self.host,
                             port=self.port,
                             user=self.user,
                             passwd=self.passwd,
                             db=self.db,
                             charset='utf8')
        return conn

    #创建数据库
    def createDatabase(self):
        conn=self.connectMysql()

        sql="create database if not exists "+self.db
        cur=conn.cursor()
        cur.execute(sql)
        cur.close()
        conn.close()

    #创建数据表
    def createTable(self,sql):
        conn=self.connectDatabase()

        cur=conn.cursor()
        cur.execute(sql)
        cur.close()
        conn.close()

    #插入数据
    def insert(self,sql,*params):
        conn=self.connectDatabase()

        cur=conn.cursor();
        cur.execute(sql,params)
        conn.commit()
        cur.close()
        conn.close()

    #更新数据
    def update(self,sql,*params):
        conn=self.connectDatabase()

        cur=conn.cursor()
        cur.execute(sql,params)
        conn.commit()
        cur.close()
        conn.close()

    #删除数据
    def delete(self,sql,*params):
        conn=self.connectDatabase()

        cur=conn.cursor()
        cur.execute(sql,params)
        conn.commit()
        cur.close()
        conn.close()


#测试数据库操作
class TestDBHelper():
    def __init__(self):
        self.dbHelper=DBHelper()

    def testCreateDatebase(self):
        self.dbHelper.createDatabase()

    def testCreateTable(self):
        sql="create table testtable(id int primary key auto_increment,name varchar(50),url varchar(200))"
        self.dbHelper.createTable(sql)

    def testInsert(self):
        sql="insert into testtable(name,url) values(%s,%s)"
        params=("test","test")
        self.dbHelper.insert(sql,*params)
    def testUpdate(self):
        sql="update testtable set name=%s,url=%s where id=%s"
        params=("update","update","1")
        self.dbHelper.update(sql,*params)

    def testDelete(self):
        sql="delete from testtable where id=%s"
        params=("1")
        self.dbHelper.delete(sql,*params)

if __name__=="__main__":
    testDBHelper=TestDBHelper()
    #testDBHelper.testCreateDatebase()  #
    #testDBHelper.testCreateTable()     #
    #testDBHelper.testInsert()          #
    #testDBHelper.testUpdate()          #
    #testDBHelper.testDelete()          #


File: liepin/liepinSpd/liepinSpd/settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for liepinSpd project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

COMPANYLIST=['7894126', '7941798', '5464493', '8280653', '8657147', '5696000', '6918711', '8801813', '7909112', '929719', '8635277', '9208490', '9427534', '7873563', '869131', '1983198', '8521820', '8441886', '9425884', '8269623', '8143143', '8144649', '8571478', '8646314', '9086358', '8361354', '8090600', '9652027', '9662729', '8029798', '8024700', '9274661', '8614537', '1852098', '845611', '7910884', '1947829', '6657987', '8463020', '8130349', '8323671', '723421', '1573297', '9582057', '1866404', '1074696', '8586065', '4811624', '857922', '7975388', '7931578', '6615613', '8243943', '682357', '8916773', '1050201', '950043', '7939262', '1730543', '9469426', '7883086', '8628525', '7868218', '8096323', '7862738', '7023768', '8862767', '9538671', '7953390', '515361', '2104592', '993518', '8212985', '1766564', '892388', '8646248', '9857531', '1043007', '8042835', '8980779', '571837', '7862722', '7935093', '8130825', '9111311', '8051561', '9107424', '856576', '7862125', '7947928', '854827', '4209085', '859352', '7931740', '7939262', '548548', '7916182', '8354065', '9740398', '8155722', '2331894', '884195', '9651734', '8534019', '7855573', '9617356', '886895', '2431058', '1939058', '8246296', '9145034', '8161625', '4450360', '540933', '4817469']

DEFAULT_REQUEST_HEADERS = {
    'Connection': 'keep-alive',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9'
}

BOT_NAME = 'liepinSpd'

MYSQL_HOST = 'rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com'
MYSQL_DBNAME = 'special_data'
MYSQL_USER = 'cn_ainvest_db'
MYSQL_PASSWD = 'cn_ainvest_sd3a1'
MYSQL_PORT = 3306

SPIDER_MODULES = ['liepinSpd.spiders']
NEWSPIDER_MODULE = 'liepinSpd.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'liepinSpd (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

USER_AGENTS = [
    "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
    "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
    "Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
    "Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0"
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"
    "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
    "Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
    "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
    "Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
    'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
    'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
    'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
    "HTC_Dream Mozilla/5.0 (Linux; U; Android 1.5; en-ca; Build/CUPCAKE) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0",
    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; sdk Build/CUPCAKE) AppleWebkit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; htc_bahamas Build/CRB17) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 2.1-update1; de-de; HTC Desire 1.19.161.5 Build/ERE27) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 1.5; de-ch; HTC Hero Build/CUPCAKE) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 2.1; en-us; HTC Legend Build/cupcake) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 1.5; de-de; HTC Magic Build/PLAT-RC33) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1 FirePHP/0.3",
    "Mozilla/5.0 (Linux; U; Android 1.6; en-us; HTC_TATTOO_A3288 Build/DRC79) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; T-Mobile G1 Build/CRB43) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari 525.20.1",
    "Mozilla/5.0 (Linux; U; Android 1.5; en-gb; T-Mobile_G2_Touch Build/CUPCAKE) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Droid Build/FRG22D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Milestone Build/ SHOLS_U2_01.03.1) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.0.1; de-de; Milestone Build/SHOLS_U2_01.14.0) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
    "Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522  (KHTML, like Gecko) Safari/419.3",
    "Mozilla/5.0 (Linux; U; Android 1.1; en-gb; dream) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
    "Mozilla/5.0 (Linux; U; Android 1.6; es-es; SonyEricssonX10i Build/R1FA016) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 1.6; en-us; SonyEricssonX10i Build/R1AA056) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
]

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'liepinSpd.middlewares.LiepinspdSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   # 'liepinSpd.middlewares.LiepinspdDownloaderMiddleware': 543,
    'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
    'liepinSpd.middlewares.MyUserAgentMiddleware': 400,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'liepinSpd.pipelines.LiepinspdPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'



File: liepin/liepinSpd/liepinSpd/items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class LiepinspdItem(scrapy.Item):
    # define the fields for your item here like:
    as_of_date = scrapy.Field()
    ticker = scrapy.Field()
    company_name = scrapy.Field()
    stage = scrapy.Field()
    size = scrapy.Field()
    city = scrapy.Field()
    industry = scrapy.Field()
    comp_clearfix = scrapy.Field()
    rate_num = scrapy.Field()
    job_count = scrapy.Field()
    registered_capital = scrapy.Field()

    spider_time = scrapy.Field()
    origin_site = scrapy.Field()




File: liepin/liepinSpd/liepinSpd/pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from twisted.enterprise import adbapi
import pymysql
import pymysql.cursors

# class LiepinspdPipeline(object):
    # def __init__(self, dbpool):
    #     self.dbpool = dbpool
    #
    # @classmethod
    # def from_settings(cls, settings):  # 函数名固定，会被scrapy调用，直接可用settings的值
    #     """
    #     数据库建立连接
    #     :param settings: 配置参数
    #     :return: 实例化参数
    #     """
    #
    #     adbparams = dict(
    #         host=settings['MYSQL_HOST'],
    #         db=settings['MYSQL_DBNAME'],
    #         user=settings['MYSQL_USER'],
    #         password=settings['MYSQL_PASSWORD'],
    #         port = settings['MYSQL_PORT'],
    #         cursorclass=pymysql.cursors.DictCursor  # 指定cursor类型
    #     )
    #     # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
    #     dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
    #     # 返回实例化参数
    #     return cls(dbpool)
    #
    # def process_item(self, item, spider):
    #     """
    #     使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
    #     """
    #     query = self.dbpool.runInteraction(self.do_insert, item)  # 指定操作方法和操作数据
    #     # 添加异常处理
    #     query.addCallback(self.handle_error)  # 处理异常
    #
    # def do_insert(self, cursor, item):
    #     # 对数据库进行插入操作，并不需要commit，twisted会自动commit
    #
    #     insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industy,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
    #     cursor.execute(insert_sql,
    #                    (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
    #                     str(item['size']), str(item['city']), str(item['industy']), str(item['comp_clearfix']),
    #                     int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),item['spider_time'],item['origin_site'],))
    # def handle_error(self, failure):
    #     if failure:
    #         # 打印错误信息
    #         print(failure)


import pymysql


class LiepinspdPipeline(object):
    """
    同步操作
    """

    def __init__(self):
        # 建立连接
        self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data')  # 有中文要存入数据库的话要加charset='utf8'
        # 创建游标
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        # sql语句
        insert_sql = """
        insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industry,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
        # 执行插入数据到数据库操作
        self.cursor.execute(insert_sql,
                            (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
                             str(item['size']), str(item['city']), str(item['industry']), str(item['comp_clearfix']),
                             int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),
                             item['spider_time'], item['origin_site'],))
        # 提交，不进行提交无法保存到数据库
        self.conn.commit()

    def close_spider(self, spider):
        # 关闭游标和连接
        self.cursor.close()
        self.conn.close()


File: liepin/liepinSpd/liepinSpd/spiders/__init__.py

# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.



File: liepin/liepinSpd/liepinSpd/spiders/lpspider.py

# !/usr/bin/env python
# -*- coding: utf-8 -*-

import scrapy
import re
from datetime import datetime
import pandas as pd
import time

from liepinSpd.items import LiepinspdItem


class LiepinSpdier(scrapy.Spider):
    name = 'liepin'
    companylist=['7894126', '7941798', '5464493', '8280653', '8657147', '5696000', '6918711', '8801813', '7909112', '929719', '8635277', '9208490', '9427534', '7873563', '869131', '1983198', '8521820', '8441886', '9425884', '8269623', '8143143', '8144649', '8571478', '8646314', '9086358', '8361354', '8090600', '9652027', '9662729', '8029798', '8024700', '9274661', '8614537', '1852098', '845611', '7910884', '1947829', '6657987', '8463020', '8130349', '8323671', '723421', '1573297', '9582057', '1866404', '1074696', '8586065', '4811624', '857922', '7975388', '7931578', '6615613', '8243943', '682357', '8916773', '1050201', '950043', '7939262', '1730543', '9469426', '7883086', '8628525', '7868218', '8096323', '7862738', '7023768', '8862767', '9538671', '7953390', '515361', '2104592', '993518', '8212985', '1766564', '892388', '8646248', '9857531', '1043007', '8042835', '8980779', '571837', '7862722', '7935093', '8130825', '9111311', '8051561', '9107424', '856576', '7862125', '7947928', '854827', '4209085', '859352', '7931740', '7939262', '548548', '7916182', '8354065', '9740398', '8155722', '2331894', '884195', '9651734', '8534019', '7855573', '9617356', '886895', '2431058', '1939058', '8246296', '9145034', '8161625', '4450360', '540933', '4817469']
    start_urls = []
    for company in companylist:
        start_urls.append(f'https://www.liepin.com/company/{company}/')

    # 公司主要基本信息
    def parse(self, response):
        # company = response.meta['company']
        text = response.text
        # print(text)
        # 抓取公司基本信息
        # try:
        company_name = response.xpath('//div[@class="name-and-welfare"]//h1/text()')[0].extract()
        # print(company_name)
        comp_sum_tag = response.xpath('//div[@class="comp-summary-tag"]/a/text()').extract()
        # 好几个
        stage=comp_sum_tag[0]
        # print(stage)
        size=comp_sum_tag[1]
        # print(size)
        city=comp_sum_tag[2]
        # print(city)
        industry=comp_sum_tag[3]
        # print(industy)
        #公司标签,list
        comp_clearfix = str(response.xpath('//ul[@class="comp-tag-list clearfix"]//span/text()').extract())
        # print(comp_clearfix)
        #简历处理率   *%转化为float
        rate_num = response.xpath('//p[@class="rate-num"]//span/text()')[0].extract()
        rate_num=int(rate_num)/100
        # print(rate_num)

        job_count = int(re.search(r'<small data-selector="total">. 共([0-9]+) 个', text).group(1))
        # print(job_count)
        #注册资本(万元)
        if '注册资本' in text and '万元人民币' in text:
            registered_capital = float(re.search(r'<li>注册资本：(.*?)万元人民币</li>', text).group(1))
        else:
            registered_capital =0.0
        # print(registered_capital)
        origin_site=re.search(r'"wapUrl":"(.*?)",', text).group(1)
        item = LiepinspdItem()
        # 匹配股票代码,判断如果股票简称全部在公司名内,则匹配股票代码
        data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
        try:
            for i in range(len(data)):
                n = 0
                for j in data.loc[i, '股票简称']:
                    if j in company_name:
                        n += 1
                if n == len(data.loc[i, '股票简称']):
                    item['ticker'] = data.loc[i, '股票代码']
                    # print(n, item['ticker'], company_name)
                # else:
                #     item['ticker'] ='未匹配'
        except BaseException as e:
            print('ticker匹配错误')

        item['as_of_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        item['company_name'] = company_name
        item['stage'] = stage
        item['size'] = size
        item['city'] = city
        item['industry'] = industry
        item['comp_clearfix'] = comp_clearfix
        item['rate_num'] = rate_num
        item['job_count'] = job_count
        item['registered_capital'] = registered_capital
        item['spider_time'] = datetime.strptime(str(datetime.now())[:10], '%Y-%m-%d').date()
        item['origin_site'] = origin_site

        yield item
        # except BaseException as e:
        #     print('error and pass')




File: liepin/liepinSpd2/run_liepin2.py

# !/usr/bin/env python
# -*- coding: utf-8 -*-

# 获取settings.py模块的设置
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from liepinSpd2.spiders.liepinJob import LiepinSpdier

settings = get_project_settings()
process = CrawlerProcess(settings=settings)

# 可以添加多个spider类
process.crawl(LiepinSpdier)

# 启动爬虫，会阻塞，直到爬取完成
process.start()


File: liepin/liepinSpd2/liepinSpd2/__init__.py




File: liepin/liepinSpd2/liepinSpd2/middlewares.py

# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
import time

from scrapy import signals
import scrapy
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
import random

from common.proxy_set import Proxies_set


class Liepinspd2SpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class Liepinspd2DownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class MyUserAgentMiddleware(UserAgentMiddleware):
    '''
    设置User-Agent
    '''

    def __init__(self, user_agent):
        self.user_agent = user_agent

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            user_agent=crawler.settings.get('USER_AGENTS')
        )

    def process_request(self, request, spider):
        agent = random.choice(self.user_agent)
        request.headers['User-Agent'] = agent
        print(agent)


# class ProxyMiddleware(object):
#     '''
#     设置Proxy
#     '''
#
#     def __init__(self, ip):
#         self.ip = ip
#
#     @classmethod
#     def from_crawler(cls, crawler):
#         return cls(ip=crawler.settings.get('PROXIES'))
#
#     def process_request(self, request, spider):
#         ip = random.choice(self.ip)
#         request.meta['proxy'] = ip


# import random
# import scrapy
# from scrapy import log


# logger = logging.getLogger()

class ProxyMiddleware(object):
    """docstring for ProxyMiddleWare"""

    def process_request(self, request, spider):
        '''对request对象加上proxy'''
        proxy = self.get_random_proxy()
        print("this is request ip:" + proxy)
        request.meta['proxy'] = proxy

    def process_response(self, request, response, spider):
        '''对返回的response处理'''
        # 如果返回的response状态不是200，重新生成当前request对象
        if response.status != 200:
            proxy = self.get_random_proxy()
            print("this is response ip:" + proxy)
            # 对当前reque加上代理
            request.meta['proxy'] = proxy
            return request
        return response

    def get_random_proxy(self):
        '''随机从文件中读取proxy'''

        while 1:
            with open('G:\workspace\common\proxies.txt', 'r') as f:
                proxies = f.readlines()
            if proxies:
                break
            else:
                time.sleep(1)
        proxy = random.choice(proxies).strip()
        return proxy



File: liepin/liepinSpd2/liepinSpd2/settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for liepinSpd2 project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'liepinSpd2'

SPIDER_MODULES = ['liepinSpd2.spiders']
NEWSPIDER_MODULE = 'liepinSpd2.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'liepinSpd2 (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

MYSQL_HOST = 'localhost'
MYSQL_DBNAME = 'day0123'
MYSQL_USER = 'root'
MYSQL_PASSWD = '123'

DEFAULT_REQUEST_HEADERS = {
    'Connection': 'keep-alive',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9'
}

USER_AGENTS = [
    "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
    "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
    "Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
    "Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
    "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
    "Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
    "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
    "Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
    'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
    'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
    'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
]

PROXIES=['27.25.194.221:9999', '113.121.147.180:9999', '111.177.170.22:9999', '116.209.53.31:9999', '111.177.189.211:9999', '111.177.188.174:9999', '111.177.181.31:9999', '211.152.33.24:48749', '125.123.142.33:9999', '125.126.192.172:9999', '58.55.206.201:9999', '58.55.202.19:9999', '171.80.174.156:9999', '183.148.133.134:9999', '111.177.172.24:9999', '124.94.199.7:9999', '121.61.1.161:9999', '58.55.192.211:9999', '183.148.133.148:9999', '59.62.164.224:9999', '111.177.165.34:9999', '111.177.178.183:9999', '121.61.25.243:9999', '27.25.196.242:9999', '117.91.232.146:9999', '111.177.178.107:9999', '111.177.188.158:9999', '111.177.179.103:9999', '111.177.181.81:9999', '183.148.133.158:9999', '110.52.235.25:9999', '111.177.187.63:9999', '111.177.172.18:9999', '111.177.178.175:9999', '116.209.54.63:9999', '183.148.140.20:9999', '116.209.52.115:9999', '117.90.2.139:9999', '111.177.177.212:9999', '119.102.189.134:9999', '119.102.188.140:9999', '119.102.188.156:9999', '121.61.2.196:9999', '49.86.180.90:9999', '219.139.141.112:9999', '111.177.189.26:9999', '111.177.191.179:9999', '122.192.174.244:9999', '111.177.167.67:9999', '125.123.139.143:9999', '125.126.210.203:9999', '125.123.140.229:9999', '171.41.84.191:9999', '111.177.185.8:9999', '110.52.235.27:9999', '123.163.117.72:9999', '111.181.35.17:9999', '113.121.146.190:9999', '111.176.29.245:9999', '116.209.58.5:9999', '111.177.175.161:9999', '113.122.169.65:9999', '121.61.2.8:808', '121.61.0.140:9999', '111.176.23.161:9999', '116.209.54.236:9999', '171.41.85.124:9999', '125.126.209.156:9999', '180.119.68.211:9999', '111.177.191.214:9999', '58.50.1.139:9999', '59.62.166.108:9999', '115.151.2.63:9999', '111.177.179.41:9999', '171.41.84.200:9999', '115.151.5.40:53128', '59.62.164.163:9999', '121.61.2.128:9999', '116.209.54.117:9999', '111.177.161.26:9999', '125.123.140.246:9999', '111.181.35.55:9999', '125.123.143.70:9999', '171.41.85.163:9999', '112.85.130.88:9999', '121.61.0.165:9999', '171.80.136.10:9999', '111.177.188.81:9999', '115.151.2.101:9999', '171.41.85.201:9999', '113.121.145.6:9999', '121.61.0.98:9999', '171.41.86.14:9999', '111.177.172.77:9999', '111.177.171.222:9999', '110.52.235.11:9999', '111.176.28.141:9999', '183.148.145.122:9999', '110.52.235.206:9999', '111.177.189.246:9999']

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'liepinSpd2.middlewares.Liepinspd2SpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
    # 'liepinSpd2.middlewares.Liepinspd2DownloaderMiddleware': 543,
    'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
    'liepinSpd2.middlewares.MyUserAgentMiddleware': 400,
    # 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': None,
    # 'liepinSpd2.middlewares.ProxyMiddleware': 125,
    # 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': None,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'liepinSpd2.pipelines.Liepinspd2Pipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'



File: liepin/liepinSpd2/liepinSpd2/items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class Liepinspd2Item(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    as_of_date = scrapy.Field()
    ticker = scrapy.Field()
    company_name = scrapy.Field()
    job_name = scrapy.Field()
    job_label = scrapy.Field()
    salary = scrapy.Field()
    city = scrapy.Field()
    education = scrapy.Field()
    work_year = scrapy.Field()
    pub_time = scrapy.Field()
    job_describe = scrapy.Field()
    # origin_site = scrapy.Field()
    function = scrapy.Field()
    spider_time = scrapy.Field()


File: liepin/liepinSpd2/liepinSpd2/pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from twisted.enterprise import adbapi
import pymysql
import pymysql.cursors
import time


# class Liepinspd2Pipeline(object):
#     def __init__(self, dbpool):
#         self.dbpool = dbpool
#
#     @classmethod
#     def from_settings(cls, settings):  # 函数名固定，会被scrapy调用，直接可用settings的值
#         """
#         数据库建立连接
#         :param settings: 配置参数
#         :return: 实例化参数
#         """
#
#         adbparams = dict(
#             host=settings['MYSQL_HOST'],
#             db=settings['MYSQL_DBNAME'],
#             user=settings['MYSQL_USER'],
#             password=settings['MYSQL_PASSWORD'],
#             cursorclass=pymysql.cursors.DictCursor  # 指定cursor类型
#         )
#         # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
#         dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
#         # 返回实例化参数
#         return cls(dbpool)
#
#     def process_item(self, item, spider):
#         """
#         使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
#         """
#         query = self.dbpool.runInteraction(self.do_insert, item)  # 指定操作方法和操作数据
#         # 添加异常处理
#         query.addCallback(self.handle_error)  # 处理异常
#
#     def do_insert(self, cursor, item):
#         # 对数据库进行插入操作，并不需要commit，twisted会自动commit
#         insert_sql = "insert into liepin_job(as_of_date,ticker,company_name,job_name,salary,city,education,work_year,pub_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
#         cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),
#                                     str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['origin_site'])))
#
#     def handle_error(self, failure):
#         if failure:
#             # 打印错误信息
#             print(failure)

class Liepinspd2Pipeline(object):
    """
    同步操作
    """

    def __init__(self):
        # 建立连接
        self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data')  # 有中文要存入数据库的话要加charset='utf8'
        # 创建游标
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        # sql语句
        insert_sql = """
        insert into job_info(as_of_date,ticker,company_name,job_name,job_label,salary,city,education,work_year,pub_time,job_describe,spider_time,function) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
        # 执行插入数据到数据库操作
        self.cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),str(item['job_label']),
                                    str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['job_describe']),item['spider_time'],str(item['function'])))
        # 提交，不进行提交无法保存到数据库
        self.conn.commit()

    def close_spider(self, spider):
        # 关闭游标和连接
        self.cursor.close()
        self.conn.close()



File: liepin/liepinSpd2/liepinSpd2/spiders/__init__.py

# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.



File: liepin/liepinSpd2/liepinSpd2/spiders/liepinJob.py

# !/usr/bin/env python
# -*- coding: utf-8 -*-

import scrapy
import re
import json
from datetime import datetime
import pandas as pd
import time
'''修改DEFAULT_CIPHERS'''
from twisted.internet.ssl import AcceptableCiphers
from scrapy.core.downloader import contextfactory
contextfactory.DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT:!DH')

from liepinSpd2.items import Liepinspd2Item


class LiepinSpdier(scrapy.Spider):
    name = 'liepin'
    companylist=['7894126', '7941798', '5464493', '8280653', '8657147', '5696000', '6918711', '8801813', '7909112', '929719', '8635277', '9208490', '9427534', '7873563', '869131', '1983198', '8521820', '8441886', '9425884', '8269623', '8143143', '8144649', '8571478', '8646314', '9086358', '8361354', '8090600', '9652027', '9662729', '8029798', '8024700', '9274661', '8614537', '1852098', '845611', '7910884', '1947829', '6657987', '8463020', '8130349', '8323671', '723421', '1573297', '9582057', '1866404', '1074696', '8586065', '4811624', '857922', '7975388', '7931578', '6615613', '8243943', '682357', '8916773', '1050201', '950043', '7939262', '1730543', '9469426', '7883086', '8628525', '7868218', '8096323', '7862738', '7023768', '8862767', '9538671', '7953390', '515361', '2104592', '993518', '8212985', '1766564', '892388', '8646248', '9857531', '1043007', '8042835', '8980779', '571837', '7862722', '7935093', '8130825', '9111311', '8051561', '9107424', '856576', '7862125', '7947928', '854827', '4209085', '859352', '7931740', '7939262', '548548', '7916182', '8354065', '9740398', '8155722', '2331894', '884195', '9651734', '8534019', '7855573', '9617356', '886895', '2431058', '1939058', '8246296', '9145034', '8161625', '4450360', '540933', '4817469']
    start_urls = []
    for company in companylist:
        start_urls.append(f'https://www.liepin.com/company/{company}/')

    # 公司主要基本信息
    def parse(self, response):
        text = response.text
        #职位总页数
        totalPage =int(re.search(r'var totalPage = ([0-9]+);', text).group(1))
        compId=re.search(r'"pcUrl":"https://www.liepin.com/company/([0-9]+)/',text).group(1)
        for i in range(1, totalPage + 1):
            print(f'第{i}页')
            url = f'https://www.liepin.com/company/{compId}/pn{i}'
            yield scrapy.Request(url,callback=self.parse_list)

    def parse_list(self, response):
        text = response.text
        urls = response.xpath('//div[@class="job-info"]/a/@href').extract()
        for url in urls:
            yield scrapy.Request(url,callback=self.parse_job)

    def parse_job(self,response):
        item=Liepinspd2Item()
        text = response.text
        as_of_date = datetime.now()
        company_name = response.xpath('//div[@class="title-info"]//a/@title')[0].extract()
        # print(company_name)
        job_name=response.xpath('//div[@class="title-info"]/h1/@title')[0].extract()
        #薪资/城市/经验/学历
        job_label=response.xpath('//li[@data-title=""]/span/text()').extract()
        salary=response.xpath('//p[@class="job-item-title"]/text()')[0].extract().strip(' \r\n')
        city=response.xpath('//p[@class="basic-infor"]//a/text()')[0].extract()
        work_year=response.xpath('//div[@class="job-qualifications"]/span/text()')[1].extract()
        education=response.xpath('//div[@class="job-qualifications"]/span/text()')[0].extract()
        pub_time=response.xpath('//p[@class="basic-infor"]/time/@title')[0].extract()
        job_describe=' '.join(response.xpath('//div[@class="content content-word"]/text()').extract())
        function=re.search(r'所属部门：</span><label>(.*?)</label></li>',text).group(1)

        data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
        try:
            for i in range(len(data)):
                n = 0
                for j in data.loc[i, '股票简称']:
                    if j in company_name:
                        n += 1
                if n == len(data.loc[i, '股票简称']):
                    item['ticker'] = data.loc[i, '股票代码']
        except BaseException as e:
            print('ticker匹配错误')

        item['as_of_date'] = as_of_date
        item['company_name'] = company_name
        item['job_name'] = job_name
        item['job_label'] = job_label
        item['salary'] = salary
        item['city'] = city
        item['education'] = education
        item['work_year'] = work_year
        item['pub_time'] = (datetime.strptime(pub_time, u"%Y年%m月%d日").date())  # 最后确定一下格式
        item['job_describe'] = job_describe
        item['function'] = function
        item['spider_time'] = datetime.strptime(str(datetime.now())[:10], '%Y-%m-%d').date()
        # item['origin_site'] = url
        # print(item['pub_time'],item['ticker'],item['company_name'])
        yield item
        # except BaseException as e:
        #     print('111error and pass')
        #     time.sleep(1)

        # company_name = response.xpath('//div[@class="name-and-welfare"]//h1/text()')[0].extract()
        # # print(company_name)
        # job_names=response.xpath('//div[@class="job-info"]/a[@class="title"]/text()').extract()
        # #薪资/城市/经验/学历
        # condition_clearfixs=response.xpath('//p[@class="condition clearfix"]/@title').extract()
        # pub_times=response.xpath('//p[@class="time-info clearfix"]/time/@title').extract()
        # urls=response.xpath('//div[@class="job-info"]/a/@href').extract()
        # for job_name, condition_clearfix, pub_time,url in zip(job_names, condition_clearfixs, pub_times,urls):
        #     # try:
        #     item['job_name']=job_name.replace('\r','').replace('\n','').replace('\t','').replace(' ','')
        #     item['salary']=condition_clearfix.split('_')[0]
        #     item['city']=condition_clearfix.split('_')[1]
        #     item['education']=condition_clearfix.split('_')[2]
        #     item['work_year']=condition_clearfix.split('_')[3]
        #     item['pub_time']=pub_time#最后确定一下格式
        #     data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
        #     try:
        #         for i in range(len(data)):
        #             n = 0
        #             for j in data.loc[i, '股票简称']:
        #                 if j in company_name:
        #                     n += 1
        #             if n == len(data.loc[i, '股票简称']):
        #                 item['ticker'] = data.loc[i, '股票代码']
        #                 print(n, item['ticker'], company_name)
        #     except BaseException as e:
        #         item['ticker'] = 'None'
        #         print('ticker匹配错误')
        #     item['as_of_date'] = as_of_date
        #     item['company_name'] = company_name
        #     item['spider_time'] = datetime.strptime(str(datetime.now())[:10], '%Y-%m-%d').date()
        #     item['origin_site'] = url
        #     print(item['pub_time'],item['ticker'],item['company_name'])
        #     yield item


File: liepin/liepinSpd_500/run_liepin1.py

# !/usr/bin/env python
# -*- coding: utf-8 -*-

# 获取settings.py模块的设置
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from liepinSpd.spiders.lpspider import LiepinSpdier

settings = get_project_settings()
process = CrawlerProcess(settings=settings)

# 可以添加多个spider类
process.crawl(LiepinSpdier)

# 启动爬虫，会阻塞，直到爬取完成
process.start()


File: liepin/liepinSpd_500/liepinSpd/__init__.py




File: liepin/liepinSpd_500/liepinSpd/middlewares.py

# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
import scrapy
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
import random

class LiepinspdSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class LiepinspdDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class MyUserAgentMiddleware(UserAgentMiddleware):
    '''
    设置User-Agent
    '''

    def __init__(self, user_agent):
        self.user_agent = user_agent

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            user_agent=crawler.settings.get('USER_AGENTS')
        )

    def process_request(self, request, spider):
        agent = random.choice(self.user_agent)
        request.headers['User-Agent'] = agent


File: liepin/liepinSpd_500/liepinSpd/dbhelper.py

import pymysql
from scrapy.utils.project import get_project_settings#引入settings配置

class DBHelper():

    def __init__(self):
        self.settings=get_project_settings()#获取settings配置数据

        self.host=self.settings['MYSQL_HOST']
        self.port=self.settings['MYSQL_PORT']
        self.user=self.settings['MYSQL_USER']
        self.passwd=self.settings['MYSQL_PASSWD']
        self.db=self.settings['MYSQL_DBNAME']
    #连接mysql
    def connectMysql(self):
        conn=pymysql.connect(host=self.host,
                             port=self.port,
                             user=self.user,
                             passwd=self.passwd,
                             charset='utf8')
        return conn
    #连接数据库
    def connectDatabase(self):
        conn=pymysql.connect(host=self.host,
                             port=self.port,
                             user=self.user,
                             passwd=self.passwd,
                             db=self.db,
                             charset='utf8')
        return conn

    #创建数据库
    def createDatabase(self):
        conn=self.connectMysql()

        sql="create database if not exists "+self.db
        cur=conn.cursor()
        cur.execute(sql)
        cur.close()
        conn.close()

    #创建数据表
    def createTable(self,sql):
        conn=self.connectDatabase()

        cur=conn.cursor()
        cur.execute(sql)
        cur.close()
        conn.close()

    #插入数据
    def insert(self,sql,*params):
        conn=self.connectDatabase()

        cur=conn.cursor();
        cur.execute(sql,params)
        conn.commit()
        cur.close()
        conn.close()

    #更新数据
    def update(self,sql,*params):
        conn=self.connectDatabase()

        cur=conn.cursor()
        cur.execute(sql,params)
        conn.commit()
        cur.close()
        conn.close()

    #删除数据
    def delete(self,sql,*params):
        conn=self.connectDatabase()

        cur=conn.cursor()
        cur.execute(sql,params)
        conn.commit()
        cur.close()
        conn.close()


#测试数据库操作
class TestDBHelper():
    def __init__(self):
        self.dbHelper=DBHelper()

    def testCreateDatebase(self):
        self.dbHelper.createDatabase()

    def testCreateTable(self):
        sql="create table testtable(id int primary key auto_increment,name varchar(50),url varchar(200))"
        self.dbHelper.createTable(sql)

    def testInsert(self):
        sql="insert into testtable(name,url) values(%s,%s)"
        params=("test","test")
        self.dbHelper.insert(sql,*params)
    def testUpdate(self):
        sql="update testtable set name=%s,url=%s where id=%s"
        params=("update","update","1")
        self.dbHelper.update(sql,*params)

    def testDelete(self):
        sql="delete from testtable where id=%s"
        params=("1")
        self.dbHelper.delete(sql,*params)

if __name__=="__main__":
    testDBHelper=TestDBHelper()
    #testDBHelper.testCreateDatebase()  #
    #testDBHelper.testCreateTable()     #
    #testDBHelper.testInsert()          #
    #testDBHelper.testUpdate()          #
    #testDBHelper.testDelete()          #


File: liepin/liepinSpd_500/liepinSpd/settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for liepinSpd project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

COMPANYLIST=['7894126', '7941798', '5464493', '8280653', '8657147', '5696000', '6918711', '8801813', '7909112', '929719', '8635277', '9208490', '9427534', '7873563', '869131', '1983198', '8521820', '8441886', '9425884', '8269623', '8143143', '8144649', '8571478', '8646314', '9086358', '8361354', '8090600', '9652027', '9662729', '8029798', '8024700', '9274661', '8614537', '1852098', '845611', '7910884', '1947829', '6657987', '8463020', '8130349', '8323671', '723421', '1573297', '9582057', '1866404', '1074696', '8586065', '4811624', '857922', '7975388', '7931578', '6615613', '8243943', '682357', '8916773', '1050201', '950043', '7939262', '1730543', '9469426', '7883086', '8628525', '7868218', '8096323', '7862738', '7023768', '8862767', '9538671', '7953390', '515361', '2104592', '993518', '8212985', '1766564', '892388', '8646248', '9857531', '1043007', '8042835', '8980779', '571837', '7862722', '7935093', '8130825', '9111311', '8051561', '9107424', '856576', '7862125', '7947928', '854827', '4209085', '859352', '7931740', '7939262', '548548', '7916182', '8354065', '9740398', '8155722', '2331894', '884195', '9651734', '8534019', '7855573', '9617356', '886895', '2431058', '1939058', '8246296', '9145034', '8161625', '4450360', '540933', '4817469']

DEFAULT_REQUEST_HEADERS = {
    'Connection': 'keep-alive',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9'
}

BOT_NAME = 'liepinSpd'

MYSQL_HOST = 'rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com'
MYSQL_DBNAME = 'special_data'
MYSQL_USER = 'cn_ainvest_db'
MYSQL_PASSWD = 'cn_ainvest_sd3a1'
MYSQL_PORT = 3306

SPIDER_MODULES = ['liepinSpd.spiders']
NEWSPIDER_MODULE = 'liepinSpd.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'liepinSpd (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

USER_AGENTS = [
    "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
    "MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
    "Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
    "Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
    "Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0"
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"
    "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
    "Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
    "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
    "Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
    'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
    'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
    'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
    "HTC_Dream Mozilla/5.0 (Linux; U; Android 1.5; en-ca; Build/CUPCAKE) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0",
    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; sdk Build/CUPCAKE) AppleWebkit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; htc_bahamas Build/CRB17) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 2.1-update1; de-de; HTC Desire 1.19.161.5 Build/ERE27) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 1.5; de-ch; HTC Hero Build/CUPCAKE) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 2.1; en-us; HTC Legend Build/cupcake) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 1.5; de-de; HTC Magic Build/PLAT-RC33) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1 FirePHP/0.3",
    "Mozilla/5.0 (Linux; U; Android 1.6; en-us; HTC_TATTOO_A3288 Build/DRC79) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
    "Mozilla/5.0 (Linux; U; Android 1.5; en-us; T-Mobile G1 Build/CRB43) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari 525.20.1",
    "Mozilla/5.0 (Linux; U; Android 1.5; en-gb; T-Mobile_G2_Touch Build/CUPCAKE) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Droid Build/FRG22D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Milestone Build/ SHOLS_U2_01.03.1) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.0.1; de-de; Milestone Build/SHOLS_U2_01.14.0) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
    "Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522  (KHTML, like Gecko) Safari/419.3",
    "Mozilla/5.0 (Linux; U; Android 1.1; en-gb; dream) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
    "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
    "Mozilla/5.0 (Linux; U; Android 1.6; es-es; SonyEricssonX10i Build/R1FA016) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
    "Mozilla/5.0 (Linux; U; Android 1.6; en-us; SonyEricssonX10i Build/R1AA056) AppleWebKit/528.5  (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
]

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'liepinSpd.middlewares.LiepinspdSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   # 'liepinSpd.middlewares.LiepinspdDownloaderMiddleware': 543,
    'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
    'liepinSpd.middlewares.MyUserAgentMiddleware': 400,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'liepinSpd.pipelines.LiepinspdPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'



File: liepin/liepinSpd_500/liepinSpd/items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class LiepinspdItem(scrapy.Item):
    # define the fields for your item here like:
    as_of_date = scrapy.Field()
    ticker = scrapy.Field()
    company_name = scrapy.Field()
    stage = scrapy.Field()
    size = scrapy.Field()
    city = scrapy.Field()
    industry = scrapy.Field()
    comp_clearfix = scrapy.Field()
    rate_num = scrapy.Field()
    job_count = scrapy.Field()
    registered_capital = scrapy.Field()

    spider_time = scrapy.Field()
    origin_site = scrapy.Field()




File: liepin/liepinSpd_500/liepinSpd/pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from twisted.enterprise import adbapi
import pymysql
import pymysql.cursors

# class LiepinspdPipeline(object):
    # def __init__(self, dbpool):
    #     self.dbpool = dbpool
    #
    # @classmethod
    # def from_settings(cls, settings):  # 函数名固定，会被scrapy调用，直接可用settings的值
    #     """
    #     数据库建立连接
    #     :param settings: 配置参数
    #     :return: 实例化参数
    #     """
    #
    #     adbparams = dict(
    #         host=settings['MYSQL_HOST'],
    #         db=settings['MYSQL_DBNAME'],
    #         user=settings['MYSQL_USER'],
    #         password=settings['MYSQL_PASSWORD'],
    #         port = settings['MYSQL_PORT'],
    #         cursorclass=pymysql.cursors.DictCursor  # 指定cursor类型
    #     )
    #     # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
    #     dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
    #     # 返回实例化参数
    #     return cls(dbpool)
    #
    # def process_item(self, item, spider):
    #     """
    #     使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
    #     """
    #     query = self.dbpool.runInteraction(self.do_insert, item)  # 指定操作方法和操作数据
    #     # 添加异常处理
    #     query.addCallback(self.handle_error)  # 处理异常
    #
    # def do_insert(self, cursor, item):
    #     # 对数据库进行插入操作，并不需要commit，twisted会自动commit
    #
    #     insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industy,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
    #     cursor.execute(insert_sql,
    #                    (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
    #                     str(item['size']), str(item['city']), str(item['industy']), str(item['comp_clearfix']),
    #                     int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),item['spider_time'],item['origin_site'],))
    # def handle_error(self, failure):
    #     if failure:
    #         # 打印错误信息
    #         print(failure)


import pymysql


class LiepinspdPipeline(object):
    """
    同步操作
    """

    def __init__(self):
        # 建立连接
        self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data')  # 有中文要存入数据库的话要加charset='utf8'
        # 创建游标
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        # sql语句
        insert_sql = """
        insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industry,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
        # 执行插入数据到数据库操作
        self.cursor.execute(insert_sql,
                            (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
                             str(item['size']), str(item['city']), str(item['industry']), str(item['comp_clearfix']),
                             int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),
                             item['spider_time'], item['origin_site'],))
        # 提交，不进行提交无法保存到数据库
        self.conn.commit()

    def close_spider(self, spider):
        # 关闭游标和连接
        self.cursor.close()
        self.conn.close()


File: liepin/liepinSpd_500/liepinSpd/spiders/__init__.py

# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.



File: liepin/liepinSpd_500/liepinSpd/spiders/lpspider.py

# !/usr/bin/env python
# -*- coding: utf-8 -*-

import scrapy
import re
from datetime import datetime
import pandas as pd
import time

from liepinSpd.items import LiepinspdItem


class LiepinSpdier(scrapy.Spider):
    name = 'liepin'
    data = pd.read_csv('G:\workspace\y2019m02\company500.csv', encoding='utf-8')
    companylist=data['股票简称']
    start_urls = []
    for company in companylist:
        start_urls.append(f'https://www.liepin.com/zhaopin/?key={company}')

    # 公司主要基本信息
    def parse(self, response):
        # company = response.meta['company']
        text = response.text
        # print(text)
        # 抓取公司基本信息
        # try:
        company_name = response.xpath('//div[@class="name-and-welfare"]//h1/text()')[0].extract()
        # print(company_name)
        comp_sum_tag = response.xpath('//div[@class="comp-summary-tag"]/a/text()').extract()
        # 好几个
        stage=comp_sum_tag[0]
        # print(stage)
        size=comp_sum_tag[1]
        # print(size)
        city=comp_sum_tag[2]
        # print(city)
        industry=comp_sum_tag[3]
        # print(industy)
        #公司标签,list
        comp_clearfix = str(response.xpath('//ul[@class="comp-tag-list clearfix"]//span/text()').extract())
        # print(comp_clearfix)
        #简历处理率   *%转化为float
        rate_num = response.xpath('//p[@class="rate-num"]//span/text()')[0].extract()
        rate_num=int(rate_num)/100
        # print(rate_num)

        job_count = int(re.search(r'<small data-selector="total">. 共([0-9]+) 个', text).group(1))
        # print(job_count)
        #注册资本(万元)
        if '注册资本' in text and '万元人民币' in text:
            registered_capital = float(re.search(r'<li>注册资本：(.*?)万元人民币</li>', text).group(1))
        else:
            registered_capital =0.0
        # print(registered_capital)
        origin_site=re.search(r'"wapUrl":"(.*?)",', text).group(1)
        item = LiepinspdItem()
        # 匹配股票代码,判断如果股票简称全部在公司名内,则匹配股票代码
        data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
        try:
            for i in range(len(data)):
                n = 0
                for j in data.loc[i, '股票简称']:
                    if j in company_name:
                        n += 1
                if n == len(data.loc[i, '股票简称']):
                    item['ticker'] = data.loc[i, '股票代码']
                    # print(n, item['ticker'], company_name)
                # else:
                #     item['ticker'] ='未匹配'
        except BaseException as e:
            print('ticker匹配错误')

        item['as_of_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        item['company_name'] = company_name
        item['stage'] = stage
        item['size'] = size
        item['city'] = city
        item['industry'] = industry
        item['comp_clearfix'] = comp_clearfix
        item['rate_num'] = rate_num
        item['job_count'] = job_count
        item['registered_capital'] = registered_capital
        item['spider_time'] = datetime.strptime(str(datetime.now())[:10], '%Y-%m-%d').date()
        item['origin_site'] = origin_site

        yield item
        # except BaseException as e:
        #     print('error and pass')




File: taobao/taobao_via_weibo.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
author : CriseLYJ
github : https://github.com/CriseLYJ/
update_time : 2019-4-2
"""

"""
淘宝登陆有时候不会跳出二维码页面，如果失败，请重新运行程序即可
"""

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


class Taobao_Spider:

    def __init__(self, username, password):
        """初始化参数"""
        url = 'https://login.taobao.com/member/login.jhtml'
        self.url = url

        options = webdriver.ChromeOptions()
        # 不加载图片,加快访问速度
        options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
        # 设置为开发者模式，避免被识别
        options.add_experimental_option('excludeSwitches',
                                        ['enable-automation'])
        self.browser = webdriver.Chrome(options=options)
        self.wait = WebDriverWait(self.browser, 10)
        # 初始化用户名
        self.username = username
        # 初始化密码
        self.password = password

    def run(self):
        """登陆接口"""
        self.browser.get(self.url)
        try:
            # 这里设置等待：等待输入框
            login_element = self.wait.until(
                EC.presence_of_element_located((By.CSS_SELECTOR, '.qrcode-login > .login-links > .forget-pwd')))
            login_element.click()

            sina_login = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.weibo-login')))
            sina_login.click()

            weibo_user = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.username > .W_input')))
            weibo_user.send_keys(self.username)

            sina_password = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.password > .W_input')))
            sina_password.send_keys(self.password)

            submit = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.btn_tip > a > span')))
            submit.click()

            taobao_name = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,
                                                                          '.site-nav-bd > ul.site-nav-bd-l > li#J_SiteNavLogin > div.site-nav-menu-hd > div.site-nav-user > a.site-nav-login-info-nick ')))
            # 登陆成功打印提示信息
            print("登陆成功：%s" % taobao_name.text)
        except Exception:
            self.browser.close()
            print("登陆失败")


if __name__ == "__main__":
    username = input("请输入你的微博用户名:")
    password = input("请输入密码:")

    spider = Taobao_Spider(username, password)
    spider.run()



File: taobao/taobao_via_username_password.py

import time

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.ui import WebDriverWait
from getpass import getpass

"""
运行前必须要做的事情：
如果直接使用webdriver，不做任何修改的话，淘宝可以断定启动的浏览器是“机器人”，而不是“死的机器”。
如果想让淘宝错误地认为启动的浏览器是"死的机器"，那么就需要修改webdriver。
我使用的是chromedriver，"perl -pi -e 's/cdc_/dog_/g' /usr/local/bin/chromedriver"是修改chromedriver的代码，直接在Terminal执行即可。执行完在运行此脚本，则可以成功登录。

这里我解释一下"perl -pi -e 's/cdc_/dog_/g' /usr/local/bin/chromedriver"，这段代码其实就是全局修改/usr/local/bin/chromedriver中的cdc_为dog_，"/usr/local/bin/chromedriver"是chromedriver所在的文件路径。
感谢https://www.jianshu.com/p/368be2cc6ca1这篇文章的作者。

######################################
- 已经修改的 webdriver 在仓库中请自行下载
- 不保证所有的版本都可用，以下是我用的版本，如果不适应，请下载对应的版本自行修改
- 另外感谢提供思路--
version: 版本 76.0.3809.100（正式版本） （64 位）
######################################
"""


class TaobaoSpider:

    def __init__(self, username, password):
        chrome_options = webdriver.ChromeOptions()
        # 不加载图片，加快访问速度
        chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
        # 设置为开发者模式，避免被识别
        chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
        self.web_driver = webdriver.Chrome(options=chrome_options)
        self.web_driver_wait = WebDriverWait(self.web_driver, 10)

        self.url = 'https://login.taobao.com/member/login.jhtml'
        self.username = username
        self.password = password

    def login(self):
        self.web_driver.get(self.url)
        try:
            # 切换为帐号密码登录
            login_method_switch = self.web_driver_wait.until(
                expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="J_QRCodeLogin"]/div[5]/a[1]')))
            login_method_switch.click()

            # 找到用户名输入框并输入
            username_input = self.web_driver_wait.until(
                expected_conditions.presence_of_element_located((By.ID, 'TPL_username_1')))
            username_input.send_keys(self.username)

            # 找到密码输入框并输入
            password_input = self.web_driver_wait.until(
                expected_conditions.presence_of_element_located((By.ID, 'TPL_password_1')))
            password_input.send_keys(self.password)

            # 找到登录按钮并点击
            login_button = self.web_driver_wait.until(
                expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="J_SubmitStatic"]')))
            login_button.click()

            # 找到名字标签并打印内容
            taobao_name_tag = self.web_driver_wait.until(expected_conditions.presence_of_element_located(
                (By.XPATH, '//*[@id="J_Col_Main"]/div/div[1]/div/div[1]/div[1]/div/div[1]/a/em')))
            print(f"登陆成功：{taobao_name_tag.text}")

            # 休息5秒钟，然后关闭浏览器
            time.sleep(5)
            self.web_driver.close()
        except Exception as e:
            print(e)
            self.web_driver.close()
            print("登陆失败")


if __name__ == "__main__":
    username = input("请输入用户名：")
    password = getpass("请输入密码：")
    spider = TaobaoSpider(username, password)
    spider.login()



File: csdn/selenium_csdn.py

# -*- coding: utf-8 -*-
# @Author: Kris
# @Mail: criselyj@163.com
# @Date:   2020-08-14 17:40:11
import os
import random
from getpass import getpass
import asyncio
from pyppeteer import launch


base_url = 'https://passport.csdn.net/login'
current_dir = os.path.dirname(os.path.realpath(__file__))
# Fix:https://github.com/miyakogi/pyppeteer/issues/183 文件权限问题。
cache_dir = os.path.join(current_dir, 'cache')
if not os.path.exists(cache_dir):
    os.mkdir(cache_dir)


class Api(object):
    def __init__(self, account, password):
        self.url = base_url
        self.account = account
        self.password = password
        self.browser = None
        self.page = None

    async def send_key(self):
        await asyncio.sleep(random.randint(2, 3))
        switch_btn = await self.page.xpath('//ul/li[@class="text-tab border-right"][2]/a')
        await switch_btn[0].click()
        input_account = await self.page.xpath('//div[@class="form-group"]/div/input[1]')
        await input_account[0].type(self.account,
                             {'delay': random.randint(100, 200) - 50})
        await self.page.type('#password-number', self.password,
                             {'delay': random.randint(100, 200) - 50})

        await self.page.click('button[data-type=account]')
        await asyncio.sleep(random.randint(5, 10))

    async def crawl(self):
        # 测试环境下 headless 设置为 False
        # 生产环境可以修改为无头浏览器
        self.browser = await launch({
            'headless': False,
            'userDataDir': cache_dir,
            'defaultViewport': {'width': 1440, 'height': 1000},
            'args': ['--no-sandbox']
        })
        self.page = await self.browser.newPage()
        await self.page.goto(self.url)

        # 伪造当前浏览状态 防止自动化工具检测
        codes = (
            "() =>{ Object.defineProperties(navigator,{ webdriver:"
            "{ get: () => false } }) }",
            "() =>{ window.navigator.chrome = { runtime: {},  }; }",
            "() =>{ Object.defineProperty(navigator, 'languages', "
            "{ get: () => ['en-US', 'en'] }); }",
            "() =>{ Object.defineProperty(navigator, 'plugins', { "
            "get: () => [1, 2, 3, 4, 5,6], }); }"
        )
        for code in codes:
            await self.page.evaluate(code)
        await self.send_key()


def main():
    print('[*] 模拟登陆 CSDN 程序启动...')
    account = input('[*] 请输入账号：')
    password = getpass('[*] 请输入密码：')
    login = Api(account, password)
    loop = asyncio.get_event_loop()
    loop.run_until_complete(login.crawl())
    

if __name__ == '__main__':
    main()



File: xiamiMusic/api.py

# -*- coding: utf-8 -*-
# @Author: Kris
# @Mail: criselyj@163.com
# @Date:   2020-08-14 17:40:11
import os
import random
import asyncio
from pyppeteer import launch

base_url = 'https://passport.xiami.com/'
current_dir = os.path.dirname(os.path.realpath(__file__))
# Fix:https://github.com/miyakogi/pyppeteer/issues/183 文件权限问题。
cache_dir = os.path.join(current_dir, 'cache')
if not os.path.exists(cache_dir):
    os.mkdir(cache_dir)


class XMLogin(object):
    url = base_url

    def __init__(self, account, password):
        self.account = account
        self.password = password
        self.browser = None
        self.page = None

    async def send_key(self):
        await self.page.click('.login-switch')
        await self.page.type('#account', self.account,
                             {'delay': random.randint(100, 200) - 50})
        await self.page.type('#password', self.password,
                             {'delay': random.randint(100, 200) - 50})

    async def slide(self):
        try:
            await self.page.hover('#captcha')
            await self.page.mouse.down()
            await self.page.mouse.move(2000, 0,
                                       {'delay': random.randint(2000, 4000)})
            await self.page.mouse.up()
        except Exception as e:
            print('error', e)
            exit(0)

    async def validate(self):
        try:
            error_element = await self.page.xpath('//div[@id="error"]')
            msg = await (
                await error_element[0].getProperty('textContent')).jsonValue()
        except Exception:
            return None
        return msg

    async def crawl(self):
        # 测试环境下 headless 设置为 False
        # 生产环境可以修改为无头浏览器
        self.browser = await launch({
            'headless': False,
            'userDataDir': cache_dir,
            'defaultViewport': {'width': 1440, 'height': 1000},
            'args': ['--no-sandbox']
        })
        self.page = await self.browser.newPage()
        await self.page.goto(self.url)
        # 伪造当前浏览状态 防止自动化工具检测
        codes = (
            "() =>{ Object.defineProperties(navigator,{ webdriver:"
            "{ get: () => false } }) }",
            "() =>{ window.navigator.chrome = { runtime: {},  }; }",
            "() =>{ Object.defineProperty(navigator, 'languages', "
            "{ get: () => ['en-US', 'en'] }); }",
            "() =>{ Object.defineProperty(navigator, 'plugins', { "
            "get: () => [1, 2, 3, 4, 5,6], }); }"
        )
        for code in codes:
            await self.page.evaluate(code)
        await self.send_key()
        await asyncio.sleep(random.randint(2, 3))
        await self.slide()

        # 登录
        await asyncio.sleep(random.randint(2, 3))
        await self.page.click('#submit')
        msg = await self.validate()
        if msg:
            print('[*] 错误信息：', msg)
            exit(0)
        print('[*] 登录成功')
        await asyncio.sleep(5)


def main():
    print('[*] 模拟登陆虾米音乐程序启动...')
    account = input('[*] 请输入账号：')
    password = input('[*] 请输入密码：')
    login = XMLogin(account, password)
    loop = asyncio.get_event_loop()
    loop.run_until_complete(login.crawl())


if __name__ == '__main__':
    main()



File: sina/sina.py

# 这里需要使用getpass模块才能使输入密码不可见
import getpass
import requests
import hashlib
import time

"""
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-7
"""


def get_login(phone, pwd):
    new_time = str(int(time.time()))
    sign = new_time + '_' + hashlib.md5((phone + pwd + new_time).encode("utf-8")).hexdigest()

    print(sign)
    url = "https://appblog.sina.com.cn/api/passport/v3_1/login.php"
    data = {
        "cookie_format": "1",
        "sign": sign,
        "pin": "e3eb41c951f264a6daa16b6e4367e829",
        "appver": "5.3.2",
        "appkey": "2546563246",
        "phone": phone,
        "entry": "app_blog",
        "pwd": pwd
    }
    headers = {
        "User-Agent": "Mozilla/5.0 (Linux; Android 5.1.1; nxt-al10 Build/LYZ28N) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36 sinablog-android/5.3.2 (Android 5.1.1; zh_CN; huawei nxt-al10/nxt-al10)",
        "Content-Type": "application/x-www-form-urlencoded; charset=utf-8"
    }
    r = requests.post(url=url, data=data, headers=headers)
    print(r.json())


if __name__ == '__main__':
    phone = input("你输入你的账号:")
    # 这里输入密码不可见
    pwd = getpass.getpass("password:")

    get_login(phone, pwd)



File: sina/spider/Ajax_weibo.py

# -*- coding: utf-8 -*-
from urllib.parse import urlencode
import requests, pymysql
from pyquery import PyQuery as pq
from selenium import webdriver
from time import sleep

# 连接数据库
connection = pymysql.connect(host='localhost',
                             port=3306,
                             user='root',
                             passwd='zkyr1006',
                             db='python',
                             charset='utf8')

cursor = connection.cursor()
sql = "USE python;"
cursor.execute(sql)
connection.commit()

base_url = 'https://m.weibo.cn/api/container/getIndex?'
headers = {
    'Host': 'm.weibo.cn',
    'Referer': 'https://m.weibo.cn/u/2145291155',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
}


def create_sheet(bozhu):
    try:
        weibo = '''
            CREATE TABLE weibo(
                ID  VARCHAR (255) NOT NULL PRIMARY KEY,
                text VARCHAR (255),
                attitudes VARCHAR (255),
                comments VARCHAR (255), 
                reposts VARCHAR (255) 
            )
        '''
        # 序号 INT  NOT NULL PRIMARY KEY AUTO_INCREMENT,
        cursor.execute(weibo)
        connection.commit()
    except:
        pass


def url_get():
    # # 自动保持cookie,不需要自己维护cookie内容
    # cookies = {}
    # s = requests.session()
    # with open('E:\example\豆瓣读书爬虫\cookie.txt')as file:
    #     raw_cookies = file.read()
    #     for line in raw_cookies.split(';'):
    #         key, value = line.split('=', 1)
    #         cookies[key] = value
    #         # 完善header
    # header = {'Upgrade-Insecure-Requests': '1',
    #           'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0',
    #           'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    #           'Accept-Encoding': 'gzip, deflate, br',
    #           'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
    #           }
    # # get请求，应答解码
    # response = s.get(url=xl_url, headers=header,cookies=cookies)
    browser = webdriver.PhantomJS()
    browser.get(url='https://m.weibo.cn/')
    wb_name = browser.find_element_by_class_name("W_input")
    wb_name.send_keys(input('输入博主ID：'))
    sleep(10)
    search = browser.find_element_by_class_name('W_ficon ficon_search S_ficon')
    search.click()
    sleep(5)
    bz_num = browser.find_element_by_class_name('name_txt')
    bz_num.click()
    sleep(5)
    # 开启了一个新页面，需要跳转到新页面
    handles = browser.window_handles
    browser.switch_to_window(handles[1])


# https://m.weibo.cn/api/container/getIndex?type=uid&value=2145291155&containerid=1076032145291155
# 拼接url
def get_page(page):
    # 查询字符串
    params = {
        'type': 'uid',
        'value': '2145291155',
        'containerid': '1076032145291155',
        'page': page
    }
    # 调用urlencode() 方法将params参数转化为 URL 的 GET请求参数
    url = base_url + urlencode(params)
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            # print(response.json())
            return response.json()
    except requests.ConnectionError as e:
        print('Error', e.args)


# 存储数据，存储到数据库
def parse_page(json):
    if json:
        items = json.get('data').get('cards')
        for index, item in enumerate(items):
            if page == 1 and index == 1:
                continue
            else:
                item = item.get('mblog')
                # weibo = {}
                # weibo['id'] = item.get('id')
                # weibo['text'] =
                # weibo['attitudes'] = item.get('attitudes_count')
                # weibo['comments'] = item.get('comments_count')
                # weibo['reposts'] = item.get('reposts_count')
                weibo = []
                weibo.append(item.get('id'))
                weibo.append(pq(item.get('text')).text())
                weibo.append(item.get('attitudes_count'))
                weibo.append(item.get('comments_count'))
                weibo.append(item.get('reposts_count'))
                # 遇见重复数据，pass，是根据主键来判断，如果是重复数据，忽略，但是报警告
                try:
                    sql = '''INSERT INTO weibo (ID,text,attitudes,comments,reposts)
                          VALUES (%s,%s,%s,%s,%s) '''
                    cursor.execute(sql, weibo)
                    connection.commit()
                except:
                    pass
            yield weibo


if __name__ == '__main__':
    for page in range(1, 17):
        json = get_page(page)
        results = parse_page(json)
        for result in results:
            print(result)

cursor.close()

# 可以爬任意指定博主所有微博，以博主名建立表，分别储存信息
# 使用selenium+PhantomJS抓取对应博主主页链接



File: sina/spider/selenium_test.py

# -*- coding: utf-8 -*-
from urllib.parse import urlencode
import requests,pymysql
from pyquery import PyQuery as pq
from selenium import webdriver
from time import sleep


cookies={}
# with open('E:\Spider\Ajax_微博\cookie.txt')as file:
#     raw_cookies=file.read()
#     for line in raw_cookies.split(';'):
#         key,value=line.split('=',1)
#         cookies[key]=value
#         # print(cookies)

webdriver.DesiredCapabilities.FIREFOX['firefox.page.settings.userAgent'] = "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"
browser = webdriver.Firefox()
browser.get(url='https://m.weibo.cn/')
sleep(2)
browser.delete_all_cookies()
browser.add_cookie({'name': '_T_WM', 'value': '52b0151e9490855bae540e40f881320f', 'path': '/',  'expiry': 1530671807, 'secure': False, 'httpOnly': True})
browser.add_cookie({'name': 'SUB', 'value': '_2A252ENWfDeRhGeVH6FQX9SzKyD6IHXVV-vvXrDV6PUJbkdANLWHmkW1NT0iViQc71Gn2lMbDYa_ioQFHVnRmZaqg', 'path': '/',  'expiry': 1559615823, 'secure': False, 'httpOnly': True})
browser.add_cookie({'name': 'SUHB', 'value': '0LcSnbMGfYxAbx', 'path': '/', 'expiry': 1559615823, 'secure': False, 'httpOnly': False})
browser.add_cookie({'name': 'SCF', 'value': 'AlYBAKMIZxonUEvbmR6JgqyYWHL1yaDreGv8vXPl1FRCO7xm1IoOxBvQbDb-ZqzXTud9qRC3AnLVu7nFx_MvHdc.', 'path': '/',  'expiry': 1843439823, 'secure': False, 'httpOnly': True})
browser.add_cookie({'name': 'SSOLoginState', 'value': '1528079823', 'path': '/','expiry': None, 'secure': False, 'httpOnly': False})
browser.add_cookie({'name': 'MLOGIN', 'value': '1', 'path': '/', 'expiry': 1528083425, 'secure': False, 'httpOnly': False})
browser.add_cookie({'name': 'M_WEIBOCN_PARAMS', 'value': 'uicode%3D20000174%26featurecode%3D20000320%26fid%3Dhotword', 'path': '/', 'expiry': 1528080425, 'secure': False, 'httpOnly': True})

sleep(2)
browser.get(url='https://m.weibo.cn/')
sleep(40)
print(browser.get_cookies())





# elem=browser.find_element_by_class_name('btn btnWhite')
# elem.click()
# username=browser.find_element_by_id('loginName')
    # username.send_keys('18401570769')


# browser.refresh()
# print(browser.page_source)
# if browser.find_element_by_class_name()



# wb_name = browser.find_element_by_class_name("W_input")
# wb_name.send_keys(input('输入博主ID：'))
# sleep(10)
# search = browser.find_element_by_class_name('W_ficon ficon_search S_ficon')
# search.click()
# sleep(5)
# bz_num = browser.find_element_by_class_name('name_txt')
# bz_num.click()
# sleep(5)
# # 开启了一个新页面，需要跳转到新页面
# handles = browser.window_handles
# browser.switch_to_window(handles[1])


<h2 align="center"><code>🎉Life is fantastic🥳!~</code></h2>

<br>
<p align="center">
    <img src="https://github.com/CriseLYJ/flask-video-streaming-recorder/blob/master/img/main.jpg?raw=true" 
        alt="Master">
</p>

<br>

<p align="center">"<i>Did you know all your doors were locked?</i>" - Riddick (The Chronicles of Riddick)</p>

<br>

<p align="center">
  <a href="https://github.com/CriseLYJ/awesome-python-login-model/tree/master">
    <img src="https://img.shields.io/badge/Branch-master-green.svg?longCache=true"
        alt="Branch">
  </a>
  <a href="https://github.com/CriseLYJ/awesome-python-login-model/stargazers">
    <img src="https://img.shields.io/github/stars/CriseLYJ/awesome-python-login-model.svg?label=Stars&style=social"
        alt="Stars">
  </a>
    <a href="https://github.com/CriseLYJ/awesome-python-login-model/network/members">
    <img src="https://img.shields.io/github/forks/CriseLYJ/awesome-python-login-model.svg?label=Forks&style=social"
        alt="Forks">
  </a>
  <a href="http://www.gnu.org/licenses/">
    <img src="https://img.shields.io/badge/License-GNU-blue.svg?longCache=true"
        alt="License">
  </a>
   <a href="https://github.com/sindresorhus/awesome">
   <img src="https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg"
        alt="Awesome">
  </a>
</p>
<br>
<div align="center">
  <sub>Created by
  <a href="https://Kr1s77.github.io/">@kris</a>
</div>

<br>

****

## 传送门
- [x] [4G 代理](https://github.com/Kr1s77/FgSurfing)
- [x] [异常处理回调，直接 hook 所有函数，和类](https://github.com/Kr1s77/abnormalities)
给个 🌟 再走吧...
    
## 💕Website login model
一些爬虫示例程序，以及模拟登陆程序,模拟登陆基于 selenium，有些模拟登录基于 js 逆向，持续更新，有问题可以直接提交 Issues，欢迎提交 PR, 测试通过可以直接 merge，文中所有程序都是使用 ``python3`` 编写 :-)

## About

模拟登陆基本采用的是直接登录或者使用selenium+webdriver的方式，有的网站直接登录难度很大，比如qq空间，bilibili等如果采用selenium就相对轻松一些。

虽然在登录的时候采用的是selenium,为了效率，我们可以在登录过后得到的cookie维护起来，然后调用requests或者scrapy等进行数据采集，这样数据采集的速度可以得到保证。

## WebDriver
[Chrome](https://chromedriver.chromium.org/)
[FireFox](https://github.com/mozilla/geckodriver/releases/)

## Completed

- [x] [虾米音乐](https://www.xiami.com/)
- [x] [Facebook](https://www.facebook.com/)
- [x] [微博网页版](http://weibo.com)
- [x] [知乎](http://zhihu.com)
- [x] [QQZone](https://qzone.qq.com/)
- [x] [CSDN](https://www.csdn.net/)
- [x] [淘宝-接口修复完成-可用](https://login.taobao.com/member/login.jhtml)
- [x] [CSDN--已重构](https://www.csdn.net/)
- [x] [Baidu](www.baidu.com)
- [x] [果壳](https://www.guokr.com/)
- [x] [JingDong 模拟登录和自动申请京东试用](https://www.jd.com/)
- [x] [163mail](https://mail.163.com/)
- [x] [拉钩](https://www.lagou.com/)
- [x] [Bilibili](https://www.bilibili.com/)
- [x] [豆瓣](https://www.douban.com/)
- [x] [豆瓣spider](https://www.douban.com/)
- [x] [Baidu](www.baidu.com)
- [x] [猎聘网](https://www.liepin.com/)
- [x] [微信网页版登录并获取好友列表](https://wx.qq.com/)
- [x] [Github](https://github.com/)
- [x] [爬取图虫相应的图片](https://tuchong.com/)
- [x] [网易云音乐](https://music.163.com/)
- [x] [糗事百科--改为协程版](https://www.qiushibaike.com/)
- [x] [百度贴吧spider](https://tieba.baidu.com/)
- [x] [百度翻译](https://fanyi.baidu.com/)

## catalogue
- [x] [虾米音乐](https://github.com/Kr1s77/awesome-python-login-model/tree/master/xiamiMusic)
- [x] [Facebook模拟登录](https://github.com/Kr1s77/awesome-python-login-model/blob/master/facebook)
- [x] [微博网页版模拟登录](https://github.com/Kr1s77/awesome-python-login-model/blob/master/sina)
- [x] [QQZone模拟登录](https://github.com/Kr1s77/awesome-python-login-model/blob/master/qqzone)
- [x] [CSDN模拟登录--已恢复](https://github.com/Kr1s77/awesome-python-login-model/blob/master/csdn)
- [x] [淘宝爬虫--重构中](https://github.com/Kr1s77/awesome-python-login-model/tree/master/taobao)
- [x] [Baidu模拟登录一](https://github.com/Kr1s77/awesome-python-login-model/tree/master/baidu)
- [x] [果壳爬虫程序](https://github.com/Kr1s77/awesome-python-login-model/tree/master/guoke)
- [x] [JingDong 模拟登录和自动申请京东试用](https://github.com/Kr1s77/awesome-python-login-model/tree/master/jd_login)
- [x] [163mail--已恢复](https://github.com/Kr1s77/awesome-python-login-model/blob/master/163email/163email.py)
- [x] [拉钩模拟登录--已失效](https://github.com/Kr1s77/awesome-python-login-model/blob/master/lagou/Lagou.py)
- [x] [Bilibili模拟登录](https://github.com/Kr1s77/awesome-python-login-model/blob/master/bilibili/bilibili.py)
- [x] [豆瓣](https://github.com/Kr1s77/awesome-python-login-model/blob/master/douban/douban.py)
- [x] [Baidu2模拟登录](https://github.com/Kr1s77/awesome-python-login-model/blob/master/baidu2/baidu.py)
- [x] [猎聘网模拟登录](https://github.com/Kr1s77/awesome-python-login-model/tree/master/liepin)
- [x] [微信网页版登录并获取好友列表](https://github.com/Kr1s77/awesome-python-login-model/blob/master/webWeixin/webWeixin.py)
- [x] [Github模拟登录两种解决方案都可行](https://github.com/Kr1s77/awesome-python-login-model/tree/master/Github)
- [x] [爬取图虫想要的图片](https://github.com/Kr1s77/awesome-python-login-model/blob/master/tuchong/tuchong.py)
- [x] [网易云音乐downloader](https://github.com/Kr1s77/awesome-python-login-model/blob/master/NeteaseCloudMusicDownload/wangyiyun_spider.py)
- [x] [糗事百科爬虫](https://github.com/Kr1s77/awesome-python-login-model/blob/master/qsbk/qiushibaike.py)
- [x] [淘宝登陆-访问](https://login.taobao.com/member/login.jhtml)


# Test

> [Please touch here to view test images](./README-Test.md)

## Informations
- 为感谢你们的支持，准备写一套免费爬虫的教程，保证你学会以后可以爬取市面上大部分的网站，[教程地址](https://github.com/CriseLYJ/-Python-crawler-starts-from-zero)

## tips of pull request 

- 欢迎大家一起来 pull request 💗

## Problems

- 关于验证码：本项目所用的方法都没有处理验证码，识别复杂验证码的难度就目前来说，还是比较大的。以我的心得来说，做爬虫最好的方式就是尽量规避验证码。
- 代码失效：由于网站策略或者样式改变，导致代码失效，请给我提issue，如果你已经解决，可以提PR，谢谢！
- 正在对部分代码进行优化。。。
- 如果该repo对大家有帮助，记得 star 哦。


## Acknowledgments

> [@deepforce](https://github.com/deepforce) | [@cclauss](https://github.com/cclauss) | [ksoeasyxiaosi](https://github.com/ksoeasyxiaosi) | [JasonJunJun](https://github.com/JasonJunJun) | [MediocrityXT](https://github.com/MediocrityXT)

- 感谢以上开发者的支持和贡献。

## 联系我
- 欢迎反馈！
- Email : criselyj@163.com

## 注意：
- 本项目仅用于学习和交流
> 欢迎任何人参与和完善：一个人可以走的很快，但是一群人却可以走的更远
