猎聘爬虫正常
This commit is contained in:
criseLYJ
@@ -0,0 +1,3 @@
|
||||
# scrapy_liepin
|
||||
|
||||
scrapy爬猎聘,通过公司名搜索公司职位
|
||||
@@ -0,0 +1,114 @@
|
||||
import pymysql
|
||||
from scrapy.utils.project import get_project_settings#引入settings配置
|
||||
|
||||
class DBHelper():
|
||||
|
||||
def __init__(self):
|
||||
self.settings=get_project_settings()#获取settings配置数据
|
||||
|
||||
self.host=self.settings['MYSQL_HOST']
|
||||
self.port=self.settings['MYSQL_PORT']
|
||||
self.user=self.settings['MYSQL_USER']
|
||||
self.passwd=self.settings['MYSQL_PASSWD']
|
||||
self.db=self.settings['MYSQL_DBNAME']
|
||||
#连接mysql
|
||||
def connectMysql(self):
|
||||
conn=pymysql.connect(host=self.host,
|
||||
port=self.port,
|
||||
user=self.user,
|
||||
passwd=self.passwd,
|
||||
charset='utf8')
|
||||
return conn
|
||||
#连接数据库
|
||||
def connectDatabase(self):
|
||||
conn=pymysql.connect(host=self.host,
|
||||
port=self.port,
|
||||
user=self.user,
|
||||
passwd=self.passwd,
|
||||
db=self.db,
|
||||
charset='utf8')
|
||||
return conn
|
||||
|
||||
#创建数据库
|
||||
def createDatabase(self):
|
||||
conn=self.connectMysql()
|
||||
|
||||
sql="create database if not exists "+self.db
|
||||
cur=conn.cursor()
|
||||
cur.execute(sql)
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
#创建数据表
|
||||
def createTable(self,sql):
|
||||
conn=self.connectDatabase()
|
||||
|
||||
cur=conn.cursor()
|
||||
cur.execute(sql)
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
#插入数据
|
||||
def insert(self,sql,*params):
|
||||
conn=self.connectDatabase()
|
||||
|
||||
cur=conn.cursor();
|
||||
cur.execute(sql,params)
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
#更新数据
|
||||
def update(self,sql,*params):
|
||||
conn=self.connectDatabase()
|
||||
|
||||
cur=conn.cursor()
|
||||
cur.execute(sql,params)
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
#删除数据
|
||||
def delete(self,sql,*params):
|
||||
conn=self.connectDatabase()
|
||||
|
||||
cur=conn.cursor()
|
||||
cur.execute(sql,params)
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
#测试数据库操作
|
||||
class TestDBHelper():
|
||||
def __init__(self):
|
||||
self.dbHelper=DBHelper()
|
||||
|
||||
def testCreateDatebase(self):
|
||||
self.dbHelper.createDatabase()
|
||||
|
||||
def testCreateTable(self):
|
||||
sql="create table testtable(id int primary key auto_increment,name varchar(50),url varchar(200))"
|
||||
self.dbHelper.createTable(sql)
|
||||
|
||||
def testInsert(self):
|
||||
sql="insert into testtable(name,url) values(%s,%s)"
|
||||
params=("test","test")
|
||||
self.dbHelper.insert(sql,*params)
|
||||
def testUpdate(self):
|
||||
sql="update testtable set name=%s,url=%s where id=%s"
|
||||
params=("update","update","1")
|
||||
self.dbHelper.update(sql,*params)
|
||||
|
||||
def testDelete(self):
|
||||
sql="delete from testtable where id=%s"
|
||||
params=("1")
|
||||
self.dbHelper.delete(sql,*params)
|
||||
|
||||
if __name__=="__main__":
|
||||
testDBHelper=TestDBHelper()
|
||||
#testDBHelper.testCreateDatebase() #
|
||||
#testDBHelper.testCreateTable() #
|
||||
#testDBHelper.testInsert() #
|
||||
#testDBHelper.testUpdate() #
|
||||
#testDBHelper.testDelete() #
|
||||
@@ -0,0 +1,27 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/items.html
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
class LiepinspdItem(scrapy.Item):
|
||||
# define the fields for your item here like:
|
||||
as_of_date = scrapy.Field()
|
||||
ticker = scrapy.Field()
|
||||
company_name = scrapy.Field()
|
||||
stage = scrapy.Field()
|
||||
size = scrapy.Field()
|
||||
city = scrapy.Field()
|
||||
industry = scrapy.Field()
|
||||
comp_clearfix = scrapy.Field()
|
||||
rate_num = scrapy.Field()
|
||||
job_count = scrapy.Field()
|
||||
registered_capital = scrapy.Field()
|
||||
|
||||
spider_time = scrapy.Field()
|
||||
origin_site = scrapy.Field()
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your spider middleware
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
from scrapy import signals
|
||||
import scrapy
|
||||
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
|
||||
import random
|
||||
|
||||
class LiepinspdSpiderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the spider middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_spider_input(self, response, spider):
|
||||
# Called for each response that goes through the spider
|
||||
# middleware and into the spider.
|
||||
|
||||
# Should return None or raise an exception.
|
||||
return None
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
# Called with the results returned from the Spider, after
|
||||
# it has processed the response.
|
||||
|
||||
# Must return an iterable of Request, dict or Item objects.
|
||||
for i in result:
|
||||
yield i
|
||||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
# Called when a spider or process_spider_input() method
|
||||
# (from other spider middleware) raises an exception.
|
||||
|
||||
# Should return either None or an iterable of Response, dict
|
||||
# or Item objects.
|
||||
pass
|
||||
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
# Called with the start requests of the spider, and works
|
||||
# similarly to the process_spider_output() method, except
|
||||
# that it doesn’t have a response associated.
|
||||
|
||||
# Must return only requests (not items).
|
||||
for r in start_requests:
|
||||
yield r
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class LiepinspdDownloaderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the downloader middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_request(self, request, spider):
|
||||
# Called for each request that goes through the downloader
|
||||
# middleware.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this request
|
||||
# - or return a Response object
|
||||
# - or return a Request object
|
||||
# - or raise IgnoreRequest: process_exception() methods of
|
||||
# installed downloader middleware will be called
|
||||
return None
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
# Called with the response returned from the downloader.
|
||||
|
||||
# Must either;
|
||||
# - return a Response object
|
||||
# - return a Request object
|
||||
# - or raise IgnoreRequest
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
# Called when a download handler or a process_request()
|
||||
# (from other downloader middleware) raises an exception.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this exception
|
||||
# - return a Response object: stops process_exception() chain
|
||||
# - return a Request object: stops process_exception() chain
|
||||
pass
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class MyUserAgentMiddleware(UserAgentMiddleware):
|
||||
'''
|
||||
设置User-Agent
|
||||
'''
|
||||
|
||||
def __init__(self, user_agent):
|
||||
self.user_agent = user_agent
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(
|
||||
user_agent=crawler.settings.get('USER_AGENTS')
|
||||
)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
agent = random.choice(self.user_agent)
|
||||
request.headers['User-Agent'] = agent
|
||||
@@ -0,0 +1,89 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
from twisted.enterprise import adbapi
|
||||
import pymysql
|
||||
import pymysql.cursors
|
||||
|
||||
# class LiepinspdPipeline(object):
|
||||
# def __init__(self, dbpool):
|
||||
# self.dbpool = dbpool
|
||||
#
|
||||
# @classmethod
|
||||
# def from_settings(cls, settings): # 函数名固定，会被scrapy调用，直接可用settings的值
|
||||
# """
|
||||
# 数据库建立连接
|
||||
# :param settings: 配置参数
|
||||
# :return: 实例化参数
|
||||
# """
|
||||
#
|
||||
# adbparams = dict(
|
||||
# host=settings['MYSQL_HOST'],
|
||||
# db=settings['MYSQL_DBNAME'],
|
||||
# user=settings['MYSQL_USER'],
|
||||
# password=settings['MYSQL_PASSWORD'],
|
||||
# port = settings['MYSQL_PORT'],
|
||||
# cursorclass=pymysql.cursors.DictCursor # 指定cursor类型
|
||||
# )
|
||||
# # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
|
||||
# dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
|
||||
# # 返回实例化参数
|
||||
# return cls(dbpool)
|
||||
#
|
||||
# def process_item(self, item, spider):
|
||||
# """
|
||||
# 使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
|
||||
# """
|
||||
# query = self.dbpool.runInteraction(self.do_insert, item) # 指定操作方法和操作数据
|
||||
# # 添加异常处理
|
||||
# query.addCallback(self.handle_error) # 处理异常
|
||||
#
|
||||
# def do_insert(self, cursor, item):
|
||||
# # 对数据库进行插入操作，并不需要commit，twisted会自动commit
|
||||
#
|
||||
# insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industy,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
|
||||
# cursor.execute(insert_sql,
|
||||
# (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
|
||||
# str(item['size']), str(item['city']), str(item['industy']), str(item['comp_clearfix']),
|
||||
# int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),item['spider_time'],item['origin_site'],))
|
||||
# def handle_error(self, failure):
|
||||
# if failure:
|
||||
# # 打印错误信息
|
||||
# print(failure)
|
||||
|
||||
|
||||
import pymysql
|
||||
|
||||
|
||||
class LiepinspdPipeline(object):
|
||||
"""
|
||||
同步操作
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# 建立连接
|
||||
self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data') # 有中文要存入数据库的话要加charset='utf8'
|
||||
# 创建游标
|
||||
self.cursor = self.conn.cursor()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
# sql语句
|
||||
insert_sql = """
|
||||
insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industry,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
||||
"""
|
||||
# 执行插入数据到数据库操作
|
||||
self.cursor.execute(insert_sql,
|
||||
(item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
|
||||
str(item['size']), str(item['city']), str(item['industry']), str(item['comp_clearfix']),
|
||||
int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),
|
||||
item['spider_time'], item['origin_site'],))
|
||||
# 提交，不进行提交无法保存到数据库
|
||||
self.conn.commit()
|
||||
|
||||
def close_spider(self, spider):
|
||||
# 关闭游标和连接
|
||||
self.cursor.close()
|
||||
self.conn.close()
|
||||
@@ -0,0 +1,211 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Scrapy settings for liepinSpd project
|
||||
#
|
||||
# For simplicity, this file contains only settings considered important or
|
||||
# commonly used. You can find more settings consulting the documentation:
|
||||
#
|
||||
# https://doc.scrapy.org/en/latest/topics/settings.html
|
||||
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
COMPANYLIST=['7894126', '7941798', '5464493', '8280653', '8657147', '5696000', '6918711', '8801813', '7909112', '929719', '8635277', '9208490', '9427534', '7873563', '869131', '1983198', '8521820', '8441886', '9425884', '8269623', '8143143', '8144649', '8571478', '8646314', '9086358', '8361354', '8090600', '9652027', '9662729', '8029798', '8024700', '9274661', '8614537', '1852098', '845611', '7910884', '1947829', '6657987', '8463020', '8130349', '8323671', '723421', '1573297', '9582057', '1866404', '1074696', '8586065', '4811624', '857922', '7975388', '7931578', '6615613', '8243943', '682357', '8916773', '1050201', '950043', '7939262', '1730543', '9469426', '7883086', '8628525', '7868218', '8096323', '7862738', '7023768', '8862767', '9538671', '7953390', '515361', '2104592', '993518', '8212985', '1766564', '892388', '8646248', '9857531', '1043007', '8042835', '8980779', '571837', '7862722', '7935093', '8130825', '9111311', '8051561', '9107424', '856576', '7862125', '7947928', '854827', '4209085', '859352', '7931740', '7939262', '548548', '7916182', '8354065', '9740398', '8155722', '2331894', '884195', '9651734', '8534019', '7855573', '9617356', '886895', '2431058', '1939058', '8246296', '9145034', '8161625', '4450360', '540933', '4817469']
|
||||
|
||||
DEFAULT_REQUEST_HEADERS = {
|
||||
'Connection': 'keep-alive',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9'
|
||||
}
|
||||
|
||||
BOT_NAME = 'liepinSpd'
|
||||
|
||||
MYSQL_HOST = 'rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com'
|
||||
MYSQL_DBNAME = 'special_data'
|
||||
MYSQL_USER = 'cn_ainvest_db'
|
||||
MYSQL_PASSWD = 'cn_ainvest_sd3a1'
|
||||
MYSQL_PORT = 3306
|
||||
|
||||
SPIDER_MODULES = ['liepinSpd.spiders']
|
||||
NEWSPIDER_MODULE = 'liepinSpd.spiders'
|
||||
|
||||
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
#USER_AGENT = 'liepinSpd (+http://www.yourdomain.com)'
|
||||
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = False
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
|
||||
"Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
|
||||
"Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0"
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
|
||||
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
|
||||
"Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
|
||||
"Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
|
||||
"Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
|
||||
'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
|
||||
'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
|
||||
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
|
||||
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
|
||||
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
|
||||
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
|
||||
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
|
||||
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
|
||||
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
|
||||
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
|
||||
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||
"HTC_Dream Mozilla/5.0 (Linux; U; Android 1.5; en-ca; Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; en-us; sdk Build/CUPCAKE) AppleWebkit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; en-us; htc_bahamas Build/CRB17) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.1-update1; de-de; HTC Desire 1.19.161.5 Build/ERE27) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; de-ch; HTC Hero Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.1; en-us; HTC Legend Build/cupcake) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; de-de; HTC Magic Build/PLAT-RC33) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1 FirePHP/0.3",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.6; en-us; HTC_TATTOO_A3288 Build/DRC79) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; en-us; T-Mobile G1 Build/CRB43) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari 525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; en-gb; T-Mobile_G2_Touch Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Droid Build/FRG22D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.0; en-us; Milestone Build/ SHOLS_U2_01.03.1) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.0.1; de-de; Milestone Build/SHOLS_U2_01.14.0) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
|
||||
"Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522 (KHTML, like Gecko) Safari/419.3",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.1; en-gb; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
|
||||
"Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.6; es-es; SonyEricssonX10i Build/R1FA016) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.6; en-us; SonyEricssonX10i Build/R1AA056) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
]
|
||||
|
||||
# Configure maximum concurrent requests performed by Scrapy (default: 16)
|
||||
#CONCURRENT_REQUESTS = 32
|
||||
|
||||
# Configure a delay for requests for the same website (default: 0)
|
||||
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
|
||||
# See also autothrottle settings and docs
|
||||
DOWNLOAD_DELAY = 3
|
||||
# The download delay setting will honor only one of:
|
||||
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
||||
#CONCURRENT_REQUESTS_PER_IP = 16
|
||||
|
||||
# Disable cookies (enabled by default)
|
||||
#COOKIES_ENABLED = False
|
||||
|
||||
# Disable Telnet Console (enabled by default)
|
||||
#TELNETCONSOLE_ENABLED = False
|
||||
|
||||
# Override the default request headers:
|
||||
|
||||
# Enable or disable spider middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
#SPIDER_MIDDLEWARES = {
|
||||
# 'liepinSpd.middlewares.LiepinspdSpiderMiddleware': 543,
|
||||
#}
|
||||
|
||||
# Enable or disable downloader middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
# 'liepinSpd.middlewares.LiepinspdDownloaderMiddleware': 543,
|
||||
'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
|
||||
'liepinSpd.middlewares.MyUserAgentMiddleware': 400,
|
||||
}
|
||||
|
||||
# Enable or disable extensions
|
||||
# See https://doc.scrapy.org/en/latest/topics/extensions.html
|
||||
#EXTENSIONS = {
|
||||
# 'scrapy.extensions.telnet.TelnetConsole': None,
|
||||
#}
|
||||
|
||||
# Configure item pipelines
|
||||
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
ITEM_PIPELINES = {
|
||||
'liepinSpd.pipelines.LiepinspdPipeline': 300,
|
||||
}
|
||||
|
||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
|
||||
#AUTOTHROTTLE_ENABLED = True
|
||||
# The initial download delay
|
||||
#AUTOTHROTTLE_START_DELAY = 5
|
||||
# The maximum download delay to be set in case of high latencies
|
||||
#AUTOTHROTTLE_MAX_DELAY = 60
|
||||
# The average number of requests Scrapy should be sending in parallel to
|
||||
# each remote server
|
||||
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||
# Enable showing throttling stats for every response received:
|
||||
#AUTOTHROTTLE_DEBUG = False
|
||||
|
||||
# Enable and configure HTTP caching (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||
#HTTPCACHE_ENABLED = True
|
||||
#HTTPCACHE_EXPIRATION_SECS = 0
|
||||
#HTTPCACHE_DIR = 'httpcache'
|
||||
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
@@ -0,0 +1,4 @@
|
||||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# Please refer to the documentation for information on how to create and manage
|
||||
# your spiders.
|
||||
@@ -0,0 +1,89 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import scrapy
|
||||
import re
|
||||
from datetime import datetime
|
||||
import pandas as pd
|
||||
import time
|
||||
|
||||
from liepinSpd.items import LiepinspdItem
|
||||
|
||||
|
||||
class LiepinSpdier(scrapy.Spider):
|
||||
name = 'liepin'
|
||||
companylist=['7894126', '7941798', '5464493', '8280653', '8657147', '5696000', '6918711', '8801813', '7909112', '929719', '8635277', '9208490', '9427534', '7873563', '869131', '1983198', '8521820', '8441886', '9425884', '8269623', '8143143', '8144649', '8571478', '8646314', '9086358', '8361354', '8090600', '9652027', '9662729', '8029798', '8024700', '9274661', '8614537', '1852098', '845611', '7910884', '1947829', '6657987', '8463020', '8130349', '8323671', '723421', '1573297', '9582057', '1866404', '1074696', '8586065', '4811624', '857922', '7975388', '7931578', '6615613', '8243943', '682357', '8916773', '1050201', '950043', '7939262', '1730543', '9469426', '7883086', '8628525', '7868218', '8096323', '7862738', '7023768', '8862767', '9538671', '7953390', '515361', '2104592', '993518', '8212985', '1766564', '892388', '8646248', '9857531', '1043007', '8042835', '8980779', '571837', '7862722', '7935093', '8130825', '9111311', '8051561', '9107424', '856576', '7862125', '7947928', '854827', '4209085', '859352', '7931740', '7939262', '548548', '7916182', '8354065', '9740398', '8155722', '2331894', '884195', '9651734', '8534019', '7855573', '9617356', '886895', '2431058', '1939058', '8246296', '9145034', '8161625', '4450360', '540933', '4817469']
|
||||
start_urls = []
|
||||
for company in companylist:
|
||||
start_urls.append(f'https://www.liepin.com/company/{company}/')
|
||||
|
||||
# 公司主要基本信息
|
||||
def parse(self, response):
|
||||
# company = response.meta['company']
|
||||
text = response.text
|
||||
# print(text)
|
||||
# 抓取公司基本信息
|
||||
# try:
|
||||
company_name = response.xpath('//div[@class="name-and-welfare"]//h1/text()')[0].extract()
|
||||
# print(company_name)
|
||||
comp_sum_tag = response.xpath('//div[@class="comp-summary-tag"]/a/text()').extract()
|
||||
# 好几个
|
||||
stage=comp_sum_tag[0]
|
||||
# print(stage)
|
||||
size=comp_sum_tag[1]
|
||||
# print(size)
|
||||
city=comp_sum_tag[2]
|
||||
# print(city)
|
||||
industry=comp_sum_tag[3]
|
||||
# print(industy)
|
||||
#公司标签,list
|
||||
comp_clearfix = str(response.xpath('//ul[@class="comp-tag-list clearfix"]//span/text()').extract())
|
||||
# print(comp_clearfix)
|
||||
#简历处理率 *%转化为float
|
||||
rate_num = response.xpath('//p[@class="rate-num"]//span/text()')[0].extract()
|
||||
rate_num=int(rate_num)/100
|
||||
# print(rate_num)
|
||||
|
||||
job_count = int(re.search(r'<small data-selector="total">. 共([0-9]+) 个', text).group(1))
|
||||
# print(job_count)
|
||||
#注册资本(万元)
|
||||
if '注册资本' in text and '万元人民币' in text:
|
||||
registered_capital = float(re.search(r'<li>注册资本：(.*?)万元人民币</li>', text).group(1))
|
||||
else:
|
||||
registered_capital =0.0
|
||||
# print(registered_capital)
|
||||
origin_site=re.search(r'"wapUrl":"(.*?)",', text).group(1)
|
||||
item = LiepinspdItem()
|
||||
# 匹配股票代码,判断如果股票简称全部在公司名内,则匹配股票代码
|
||||
data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
|
||||
try:
|
||||
for i in range(len(data)):
|
||||
n = 0
|
||||
for j in data.loc[i, '股票简称']:
|
||||
if j in company_name:
|
||||
n += 1
|
||||
if n == len(data.loc[i, '股票简称']):
|
||||
item['ticker'] = data.loc[i, '股票代码']
|
||||
# print(n, item['ticker'], company_name)
|
||||
# else:
|
||||
# item['ticker'] ='未匹配'
|
||||
except BaseException as e:
|
||||
print('ticker匹配错误')
|
||||
|
||||
item['as_of_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
item['company_name'] = company_name
|
||||
item['stage'] = stage
|
||||
item['size'] = size
|
||||
item['city'] = city
|
||||
item['industry'] = industry
|
||||
item['comp_clearfix'] = comp_clearfix
|
||||
item['rate_num'] = rate_num
|
||||
item['job_count'] = job_count
|
||||
item['registered_capital'] = registered_capital
|
||||
item['spider_time'] = datetime.strptime(str(datetime.now())[:10], '%Y-%m-%d').date()
|
||||
item['origin_site'] = origin_site
|
||||
|
||||
yield item
|
||||
# except BaseException as e:
|
||||
# print('error and pass')
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# 获取settings.py模块的设置
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
from liepinSpd.spiders.lpspider import LiepinSpdier
|
||||
|
||||
settings = get_project_settings()
|
||||
process = CrawlerProcess(settings=settings)
|
||||
|
||||
# 可以添加多个spider类
|
||||
process.crawl(LiepinSpdier)
|
||||
|
||||
# 启动爬虫，会阻塞，直到爬取完成
|
||||
process.start()
|
||||
@@ -0,0 +1,11 @@
|
||||
# Automatically created by: scrapy startproject
|
||||
#
|
||||
# For more information about the [deploy] section see:
|
||||
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
|
||||
[settings]
|
||||
default = liepinSpd.settings
|
||||
|
||||
[deploy]
|
||||
#url = http://localhost:6800/
|
||||
project = liepinSpd
|
||||
@@ -0,0 +1,27 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/items.html
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
class Liepinspd2Item(scrapy.Item):
|
||||
# define the fields for your item here like:
|
||||
# name = scrapy.Field()
|
||||
as_of_date = scrapy.Field()
|
||||
ticker = scrapy.Field()
|
||||
company_name = scrapy.Field()
|
||||
job_name = scrapy.Field()
|
||||
job_label = scrapy.Field()
|
||||
salary = scrapy.Field()
|
||||
city = scrapy.Field()
|
||||
education = scrapy.Field()
|
||||
work_year = scrapy.Field()
|
||||
pub_time = scrapy.Field()
|
||||
job_describe = scrapy.Field()
|
||||
# origin_site = scrapy.Field()
|
||||
function = scrapy.Field()
|
||||
spider_time = scrapy.Field()
|
||||
@@ -0,0 +1,187 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your spider middleware
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
import time
|
||||
|
||||
from scrapy import signals
|
||||
import scrapy
|
||||
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
|
||||
import random
|
||||
|
||||
from common.proxy_set import Proxies_set
|
||||
|
||||
|
||||
class Liepinspd2SpiderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the spider middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_spider_input(self, response, spider):
|
||||
# Called for each response that goes through the spider
|
||||
# middleware and into the spider.
|
||||
|
||||
# Should return None or raise an exception.
|
||||
return None
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
# Called with the results returned from the Spider, after
|
||||
# it has processed the response.
|
||||
|
||||
# Must return an iterable of Request, dict or Item objects.
|
||||
for i in result:
|
||||
yield i
|
||||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
# Called when a spider or process_spider_input() method
|
||||
# (from other spider middleware) raises an exception.
|
||||
|
||||
# Should return either None or an iterable of Response, dict
|
||||
# or Item objects.
|
||||
pass
|
||||
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
# Called with the start requests of the spider, and works
|
||||
# similarly to the process_spider_output() method, except
|
||||
# that it doesn’t have a response associated.
|
||||
|
||||
# Must return only requests (not items).
|
||||
for r in start_requests:
|
||||
yield r
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class Liepinspd2DownloaderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the downloader middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_request(self, request, spider):
|
||||
# Called for each request that goes through the downloader
|
||||
# middleware.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this request
|
||||
# - or return a Response object
|
||||
# - or return a Request object
|
||||
# - or raise IgnoreRequest: process_exception() methods of
|
||||
# installed downloader middleware will be called
|
||||
return None
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
# Called with the response returned from the downloader.
|
||||
|
||||
# Must either;
|
||||
# - return a Response object
|
||||
# - return a Request object
|
||||
# - or raise IgnoreRequest
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
# Called when a download handler or a process_request()
|
||||
# (from other downloader middleware) raises an exception.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this exception
|
||||
# - return a Response object: stops process_exception() chain
|
||||
# - return a Request object: stops process_exception() chain
|
||||
pass
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class MyUserAgentMiddleware(UserAgentMiddleware):
|
||||
'''
|
||||
设置User-Agent
|
||||
'''
|
||||
|
||||
def __init__(self, user_agent):
|
||||
self.user_agent = user_agent
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(
|
||||
user_agent=crawler.settings.get('USER_AGENTS')
|
||||
)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
agent = random.choice(self.user_agent)
|
||||
request.headers['User-Agent'] = agent
|
||||
print(agent)
|
||||
|
||||
|
||||
# class ProxyMiddleware(object):
|
||||
# '''
|
||||
# 设置Proxy
|
||||
# '''
|
||||
#
|
||||
# def __init__(self, ip):
|
||||
# self.ip = ip
|
||||
#
|
||||
# @classmethod
|
||||
# def from_crawler(cls, crawler):
|
||||
# return cls(ip=crawler.settings.get('PROXIES'))
|
||||
#
|
||||
# def process_request(self, request, spider):
|
||||
# ip = random.choice(self.ip)
|
||||
# request.meta['proxy'] = ip
|
||||
|
||||
|
||||
# import random
|
||||
# import scrapy
|
||||
# from scrapy import log
|
||||
|
||||
|
||||
# logger = logging.getLogger()
|
||||
|
||||
class ProxyMiddleware(object):
|
||||
"""docstring for ProxyMiddleWare"""
|
||||
|
||||
def process_request(self, request, spider):
|
||||
'''对request对象加上proxy'''
|
||||
proxy = self.get_random_proxy()
|
||||
print("this is request ip:" + proxy)
|
||||
request.meta['proxy'] = proxy
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
'''对返回的response处理'''
|
||||
# 如果返回的response状态不是200，重新生成当前request对象
|
||||
if response.status != 200:
|
||||
proxy = self.get_random_proxy()
|
||||
print("this is response ip:" + proxy)
|
||||
# 对当前reque加上代理
|
||||
request.meta['proxy'] = proxy
|
||||
return request
|
||||
return response
|
||||
|
||||
def get_random_proxy(self):
|
||||
'''随机从文件中读取proxy'''
|
||||
|
||||
while 1:
|
||||
with open('G:\workspace\common\proxies.txt', 'r') as f:
|
||||
proxies = f.readlines()
|
||||
if proxies:
|
||||
break
|
||||
else:
|
||||
time.sleep(1)
|
||||
proxy = random.choice(proxies).strip()
|
||||
return proxy
|
||||
@@ -0,0 +1,81 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
from twisted.enterprise import adbapi
|
||||
import pymysql
|
||||
import pymysql.cursors
|
||||
import time
|
||||
|
||||
|
||||
# class Liepinspd2Pipeline(object):
|
||||
# def __init__(self, dbpool):
|
||||
# self.dbpool = dbpool
|
||||
#
|
||||
# @classmethod
|
||||
# def from_settings(cls, settings): # 函数名固定，会被scrapy调用，直接可用settings的值
|
||||
# """
|
||||
# 数据库建立连接
|
||||
# :param settings: 配置参数
|
||||
# :return: 实例化参数
|
||||
# """
|
||||
#
|
||||
# adbparams = dict(
|
||||
# host=settings['MYSQL_HOST'],
|
||||
# db=settings['MYSQL_DBNAME'],
|
||||
# user=settings['MYSQL_USER'],
|
||||
# password=settings['MYSQL_PASSWORD'],
|
||||
# cursorclass=pymysql.cursors.DictCursor # 指定cursor类型
|
||||
# )
|
||||
# # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
|
||||
# dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
|
||||
# # 返回实例化参数
|
||||
# return cls(dbpool)
|
||||
#
|
||||
# def process_item(self, item, spider):
|
||||
# """
|
||||
# 使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
|
||||
# """
|
||||
# query = self.dbpool.runInteraction(self.do_insert, item) # 指定操作方法和操作数据
|
||||
# # 添加异常处理
|
||||
# query.addCallback(self.handle_error) # 处理异常
|
||||
#
|
||||
# def do_insert(self, cursor, item):
|
||||
# # 对数据库进行插入操作，并不需要commit，twisted会自动commit
|
||||
# insert_sql = "insert into liepin_job(as_of_date,ticker,company_name,job_name,salary,city,education,work_year,pub_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
|
||||
# cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),
|
||||
# str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['origin_site'])))
|
||||
#
|
||||
# def handle_error(self, failure):
|
||||
# if failure:
|
||||
# # 打印错误信息
|
||||
# print(failure)
|
||||
|
||||
class Liepinspd2Pipeline(object):
|
||||
"""
|
||||
同步操作
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# 建立连接
|
||||
self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data') # 有中文要存入数据库的话要加charset='utf8'
|
||||
# 创建游标
|
||||
self.cursor = self.conn.cursor()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
# sql语句
|
||||
insert_sql = """
|
||||
insert into job_info(as_of_date,ticker,company_name,job_name,job_label,salary,city,education,work_year,pub_time,job_describe,spider_time,function) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
||||
"""
|
||||
# 执行插入数据到数据库操作
|
||||
self.cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),str(item['job_label']),
|
||||
str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['job_describe']),item['spider_time'],str(item['function'])))
|
||||
# 提交，不进行提交无法保存到数据库
|
||||
self.conn.commit()
|
||||
|
||||
def close_spider(self, spider):
|
||||
# 关闭游标和连接
|
||||
self.cursor.close()
|
||||
self.conn.close()
|
||||
@@ -0,0 +1,181 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Scrapy settings for liepinSpd2 project
|
||||
#
|
||||
# For simplicity, this file contains only settings considered important or
|
||||
# commonly used. You can find more settings consulting the documentation:
|
||||
#
|
||||
# https://doc.scrapy.org/en/latest/topics/settings.html
|
||||
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
BOT_NAME = 'liepinSpd2'
|
||||
|
||||
SPIDER_MODULES = ['liepinSpd2.spiders']
|
||||
NEWSPIDER_MODULE = 'liepinSpd2.spiders'
|
||||
|
||||
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
#USER_AGENT = 'liepinSpd2 (+http://www.yourdomain.com)'
|
||||
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = False
|
||||
|
||||
MYSQL_HOST = 'localhost'
|
||||
MYSQL_DBNAME = 'day0123'
|
||||
MYSQL_USER = 'root'
|
||||
MYSQL_PASSWD = '123'
|
||||
|
||||
DEFAULT_REQUEST_HEADERS = {
|
||||
'Connection': 'keep-alive',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9'
|
||||
}
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
|
||||
"Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
|
||||
"Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
|
||||
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
|
||||
"Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
|
||||
"Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
|
||||
"Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
|
||||
'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
|
||||
'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
|
||||
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
|
||||
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
|
||||
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
|
||||
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
|
||||
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
|
||||
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
|
||||
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
|
||||
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
|
||||
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||
]
|
||||
|
||||
PROXIES=['27.25.194.221:9999', '113.121.147.180:9999', '111.177.170.22:9999', '116.209.53.31:9999', '111.177.189.211:9999', '111.177.188.174:9999', '111.177.181.31:9999', '211.152.33.24:48749', '125.123.142.33:9999', '125.126.192.172:9999', '58.55.206.201:9999', '58.55.202.19:9999', '171.80.174.156:9999', '183.148.133.134:9999', '111.177.172.24:9999', '124.94.199.7:9999', '121.61.1.161:9999', '58.55.192.211:9999', '183.148.133.148:9999', '59.62.164.224:9999', '111.177.165.34:9999', '111.177.178.183:9999', '121.61.25.243:9999', '27.25.196.242:9999', '117.91.232.146:9999', '111.177.178.107:9999', '111.177.188.158:9999', '111.177.179.103:9999', '111.177.181.81:9999', '183.148.133.158:9999', '110.52.235.25:9999', '111.177.187.63:9999', '111.177.172.18:9999', '111.177.178.175:9999', '116.209.54.63:9999', '183.148.140.20:9999', '116.209.52.115:9999', '117.90.2.139:9999', '111.177.177.212:9999', '119.102.189.134:9999', '119.102.188.140:9999', '119.102.188.156:9999', '121.61.2.196:9999', '49.86.180.90:9999', '219.139.141.112:9999', '111.177.189.26:9999', '111.177.191.179:9999', '122.192.174.244:9999', '111.177.167.67:9999', '125.123.139.143:9999', '125.126.210.203:9999', '125.123.140.229:9999', '171.41.84.191:9999', '111.177.185.8:9999', '110.52.235.27:9999', '123.163.117.72:9999', '111.181.35.17:9999', '113.121.146.190:9999', '111.176.29.245:9999', '116.209.58.5:9999', '111.177.175.161:9999', '113.122.169.65:9999', '121.61.2.8:808', '121.61.0.140:9999', '111.176.23.161:9999', '116.209.54.236:9999', '171.41.85.124:9999', '125.126.209.156:9999', '180.119.68.211:9999', '111.177.191.214:9999', '58.50.1.139:9999', '59.62.166.108:9999', '115.151.2.63:9999', '111.177.179.41:9999', '171.41.84.200:9999', '115.151.5.40:53128', '59.62.164.163:9999', '121.61.2.128:9999', '116.209.54.117:9999', '111.177.161.26:9999', '125.123.140.246:9999', '111.181.35.55:9999', '125.123.143.70:9999', '171.41.85.163:9999', '112.85.130.88:9999', '121.61.0.165:9999', '171.80.136.10:9999', '111.177.188.81:9999', '115.151.2.101:9999', '171.41.85.201:9999', '113.121.145.6:9999', '121.61.0.98:9999', '171.41.86.14:9999', '111.177.172.77:9999', '111.177.171.222:9999', '110.52.235.11:9999', '111.176.28.141:9999', '183.148.145.122:9999', '110.52.235.206:9999', '111.177.189.246:9999']
|
||||
|
||||
# Configure maximum concurrent requests performed by Scrapy (default: 16)
|
||||
#CONCURRENT_REQUESTS = 32
|
||||
|
||||
# Configure a delay for requests for the same website (default: 0)
|
||||
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
|
||||
# See also autothrottle settings and docs
|
||||
DOWNLOAD_DELAY = 3
|
||||
# The download delay setting will honor only one of:
|
||||
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
||||
#CONCURRENT_REQUESTS_PER_IP = 16
|
||||
|
||||
# Disable cookies (enabled by default)
|
||||
#COOKIES_ENABLED = False
|
||||
|
||||
# Disable Telnet Console (enabled by default)
|
||||
#TELNETCONSOLE_ENABLED = False
|
||||
|
||||
# Override the default request headers:
|
||||
|
||||
# Enable or disable spider middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
#SPIDER_MIDDLEWARES = {
|
||||
# 'liepinSpd2.middlewares.Liepinspd2SpiderMiddleware': 543,
|
||||
#}
|
||||
|
||||
# Enable or disable downloader middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
# 'liepinSpd2.middlewares.Liepinspd2DownloaderMiddleware': 543,
|
||||
'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
|
||||
'liepinSpd2.middlewares.MyUserAgentMiddleware': 400,
|
||||
# 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': None,
|
||||
# 'liepinSpd2.middlewares.ProxyMiddleware': 125,
|
||||
# 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': None,
|
||||
}
|
||||
|
||||
# Enable or disable extensions
|
||||
# See https://doc.scrapy.org/en/latest/topics/extensions.html
|
||||
#EXTENSIONS = {
|
||||
# 'scrapy.extensions.telnet.TelnetConsole': None,
|
||||
#}
|
||||
|
||||
# Configure item pipelines
|
||||
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
ITEM_PIPELINES = {
|
||||
'liepinSpd2.pipelines.Liepinspd2Pipeline': 300,
|
||||
}
|
||||
|
||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
|
||||
#AUTOTHROTTLE_ENABLED = True
|
||||
# The initial download delay
|
||||
#AUTOTHROTTLE_START_DELAY = 5
|
||||
# The maximum download delay to be set in case of high latencies
|
||||
#AUTOTHROTTLE_MAX_DELAY = 60
|
||||
# The average number of requests Scrapy should be sending in parallel to
|
||||
# each remote server
|
||||
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||
# Enable showing throttling stats for every response received:
|
||||
#AUTOTHROTTLE_DEBUG = False
|
||||
|
||||
# Enable and configure HTTP caching (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||
#HTTPCACHE_ENABLED = True
|
||||
#HTTPCACHE_EXPIRATION_SECS = 0
|
||||
#HTTPCACHE_DIR = 'httpcache'
|
||||
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
@@ -0,0 +1,4 @@
|
||||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# Please refer to the documentation for information on how to create and manage
|
||||
# your spiders.
|
||||
@@ -0,0 +1,123 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import scrapy
|
||||
import re
|
||||
import json
|
||||
from datetime import datetime
|
||||
import pandas as pd
|
||||
import time
|
||||
'''修改DEFAULT_CIPHERS'''
|
||||
from twisted.internet.ssl import AcceptableCiphers
|
||||
from scrapy.core.downloader import contextfactory
|
||||
contextfactory.DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT:!DH')
|
||||
|
||||
from liepinSpd2.items import Liepinspd2Item
|
||||
|
||||
|
||||
class LiepinSpdier(scrapy.Spider):
|
||||
name = 'liepin'
|
||||
companylist=['7894126', '7941798', '5464493', '8280653', '8657147', '5696000', '6918711', '8801813', '7909112', '929719', '8635277', '9208490', '9427534', '7873563', '869131', '1983198', '8521820', '8441886', '9425884', '8269623', '8143143', '8144649', '8571478', '8646314', '9086358', '8361354', '8090600', '9652027', '9662729', '8029798', '8024700', '9274661', '8614537', '1852098', '845611', '7910884', '1947829', '6657987', '8463020', '8130349', '8323671', '723421', '1573297', '9582057', '1866404', '1074696', '8586065', '4811624', '857922', '7975388', '7931578', '6615613', '8243943', '682357', '8916773', '1050201', '950043', '7939262', '1730543', '9469426', '7883086', '8628525', '7868218', '8096323', '7862738', '7023768', '8862767', '9538671', '7953390', '515361', '2104592', '993518', '8212985', '1766564', '892388', '8646248', '9857531', '1043007', '8042835', '8980779', '571837', '7862722', '7935093', '8130825', '9111311', '8051561', '9107424', '856576', '7862125', '7947928', '854827', '4209085', '859352', '7931740', '7939262', '548548', '7916182', '8354065', '9740398', '8155722', '2331894', '884195', '9651734', '8534019', '7855573', '9617356', '886895', '2431058', '1939058', '8246296', '9145034', '8161625', '4450360', '540933', '4817469']
|
||||
start_urls = []
|
||||
for company in companylist:
|
||||
start_urls.append(f'https://www.liepin.com/company/{company}/')
|
||||
|
||||
# 公司主要基本信息
|
||||
def parse(self, response):
|
||||
text = response.text
|
||||
#职位总页数
|
||||
totalPage =int(re.search(r'var totalPage = ([0-9]+);', text).group(1))
|
||||
compId=re.search(r'"pcUrl":"https://www.liepin.com/company/([0-9]+)/',text).group(1)
|
||||
for i in range(1, totalPage + 1):
|
||||
print(f'第{i}页')
|
||||
url = f'https://www.liepin.com/company/{compId}/pn{i}'
|
||||
yield scrapy.Request(url,callback=self.parse_list)
|
||||
|
||||
def parse_list(self, response):
|
||||
text = response.text
|
||||
urls = response.xpath('//div[@class="job-info"]/a/@href').extract()
|
||||
for url in urls:
|
||||
yield scrapy.Request(url,callback=self.parse_job)
|
||||
|
||||
def parse_job(self,response):
|
||||
item=Liepinspd2Item()
|
||||
text = response.text
|
||||
as_of_date = datetime.now()
|
||||
company_name = response.xpath('//div[@class="title-info"]//a/@title')[0].extract()
|
||||
# print(company_name)
|
||||
job_name=response.xpath('//div[@class="title-info"]/h1/@title')[0].extract()
|
||||
#薪资/城市/经验/学历
|
||||
job_label=response.xpath('//li[@data-title=""]/span/text()').extract()
|
||||
salary=response.xpath('//p[@class="job-item-title"]/text()')[0].extract().strip(' \r\n')
|
||||
city=response.xpath('//p[@class="basic-infor"]//a/text()')[0].extract()
|
||||
work_year=response.xpath('//div[@class="job-qualifications"]/span/text()')[1].extract()
|
||||
education=response.xpath('//div[@class="job-qualifications"]/span/text()')[0].extract()
|
||||
pub_time=response.xpath('//p[@class="basic-infor"]/time/@title')[0].extract()
|
||||
job_describe=' '.join(response.xpath('//div[@class="content content-word"]/text()').extract())
|
||||
function=re.search(r'所属部门：</span><label>(.*?)</label></li>',text).group(1)
|
||||
|
||||
data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
|
||||
try:
|
||||
for i in range(len(data)):
|
||||
n = 0
|
||||
for j in data.loc[i, '股票简称']:
|
||||
if j in company_name:
|
||||
n += 1
|
||||
if n == len(data.loc[i, '股票简称']):
|
||||
item['ticker'] = data.loc[i, '股票代码']
|
||||
except BaseException as e:
|
||||
print('ticker匹配错误')
|
||||
|
||||
item['as_of_date'] = as_of_date
|
||||
item['company_name'] = company_name
|
||||
item['job_name'] = job_name
|
||||
item['job_label'] = job_label
|
||||
item['salary'] = salary
|
||||
item['city'] = city
|
||||
item['education'] = education
|
||||
item['work_year'] = work_year
|
||||
item['pub_time'] = (datetime.strptime(pub_time, u"%Y年%m月%d日").date()) # 最后确定一下格式
|
||||
item['job_describe'] = job_describe
|
||||
item['function'] = function
|
||||
item['spider_time'] = datetime.strptime(str(datetime.now())[:10], '%Y-%m-%d').date()
|
||||
# item['origin_site'] = url
|
||||
# print(item['pub_time'],item['ticker'],item['company_name'])
|
||||
yield item
|
||||
# except BaseException as e:
|
||||
# print('111error and pass')
|
||||
# time.sleep(1)
|
||||
|
||||
# company_name = response.xpath('//div[@class="name-and-welfare"]//h1/text()')[0].extract()
|
||||
# # print(company_name)
|
||||
# job_names=response.xpath('//div[@class="job-info"]/a[@class="title"]/text()').extract()
|
||||
# #薪资/城市/经验/学历
|
||||
# condition_clearfixs=response.xpath('//p[@class="condition clearfix"]/@title').extract()
|
||||
# pub_times=response.xpath('//p[@class="time-info clearfix"]/time/@title').extract()
|
||||
# urls=response.xpath('//div[@class="job-info"]/a/@href').extract()
|
||||
# for job_name, condition_clearfix, pub_time,url in zip(job_names, condition_clearfixs, pub_times,urls):
|
||||
# # try:
|
||||
# item['job_name']=job_name.replace('\r','').replace('\n','').replace('\t','').replace(' ','')
|
||||
# item['salary']=condition_clearfix.split('_')[0]
|
||||
# item['city']=condition_clearfix.split('_')[1]
|
||||
# item['education']=condition_clearfix.split('_')[2]
|
||||
# item['work_year']=condition_clearfix.split('_')[3]
|
||||
# item['pub_time']=pub_time#最后确定一下格式
|
||||
# data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
|
||||
# try:
|
||||
# for i in range(len(data)):
|
||||
# n = 0
|
||||
# for j in data.loc[i, '股票简称']:
|
||||
# if j in company_name:
|
||||
# n += 1
|
||||
# if n == len(data.loc[i, '股票简称']):
|
||||
# item['ticker'] = data.loc[i, '股票代码']
|
||||
# print(n, item['ticker'], company_name)
|
||||
# except BaseException as e:
|
||||
# item['ticker'] = 'None'
|
||||
# print('ticker匹配错误')
|
||||
# item['as_of_date'] = as_of_date
|
||||
# item['company_name'] = company_name
|
||||
# item['spider_time'] = datetime.strptime(str(datetime.now())[:10], '%Y-%m-%d').date()
|
||||
# item['origin_site'] = url
|
||||
# print(item['pub_time'],item['ticker'],item['company_name'])
|
||||
# yield item
|
||||
@@ -0,0 +1,17 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# 获取settings.py模块的设置
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
from liepinSpd2.spiders.liepinJob import LiepinSpdier
|
||||
|
||||
settings = get_project_settings()
|
||||
process = CrawlerProcess(settings=settings)
|
||||
|
||||
# 可以添加多个spider类
|
||||
process.crawl(LiepinSpdier)
|
||||
|
||||
# 启动爬虫，会阻塞，直到爬取完成
|
||||
process.start()
|
||||
@@ -0,0 +1,11 @@
|
||||
# Automatically created by: scrapy startproject
|
||||
#
|
||||
# For more information about the [deploy] section see:
|
||||
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
|
||||
[settings]
|
||||
default = liepinSpd2.settings
|
||||
|
||||
[deploy]
|
||||
#url = http://localhost:6800/
|
||||
project = liepinSpd2
|
||||
@@ -0,0 +1,114 @@
|
||||
import pymysql
|
||||
from scrapy.utils.project import get_project_settings#引入settings配置
|
||||
|
||||
class DBHelper():
|
||||
|
||||
def __init__(self):
|
||||
self.settings=get_project_settings()#获取settings配置数据
|
||||
|
||||
self.host=self.settings['MYSQL_HOST']
|
||||
self.port=self.settings['MYSQL_PORT']
|
||||
self.user=self.settings['MYSQL_USER']
|
||||
self.passwd=self.settings['MYSQL_PASSWD']
|
||||
self.db=self.settings['MYSQL_DBNAME']
|
||||
#连接mysql
|
||||
def connectMysql(self):
|
||||
conn=pymysql.connect(host=self.host,
|
||||
port=self.port,
|
||||
user=self.user,
|
||||
passwd=self.passwd,
|
||||
charset='utf8')
|
||||
return conn
|
||||
#连接数据库
|
||||
def connectDatabase(self):
|
||||
conn=pymysql.connect(host=self.host,
|
||||
port=self.port,
|
||||
user=self.user,
|
||||
passwd=self.passwd,
|
||||
db=self.db,
|
||||
charset='utf8')
|
||||
return conn
|
||||
|
||||
#创建数据库
|
||||
def createDatabase(self):
|
||||
conn=self.connectMysql()
|
||||
|
||||
sql="create database if not exists "+self.db
|
||||
cur=conn.cursor()
|
||||
cur.execute(sql)
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
#创建数据表
|
||||
def createTable(self,sql):
|
||||
conn=self.connectDatabase()
|
||||
|
||||
cur=conn.cursor()
|
||||
cur.execute(sql)
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
#插入数据
|
||||
def insert(self,sql,*params):
|
||||
conn=self.connectDatabase()
|
||||
|
||||
cur=conn.cursor();
|
||||
cur.execute(sql,params)
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
#更新数据
|
||||
def update(self,sql,*params):
|
||||
conn=self.connectDatabase()
|
||||
|
||||
cur=conn.cursor()
|
||||
cur.execute(sql,params)
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
#删除数据
|
||||
def delete(self,sql,*params):
|
||||
conn=self.connectDatabase()
|
||||
|
||||
cur=conn.cursor()
|
||||
cur.execute(sql,params)
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
#测试数据库操作
|
||||
class TestDBHelper():
|
||||
def __init__(self):
|
||||
self.dbHelper=DBHelper()
|
||||
|
||||
def testCreateDatebase(self):
|
||||
self.dbHelper.createDatabase()
|
||||
|
||||
def testCreateTable(self):
|
||||
sql="create table testtable(id int primary key auto_increment,name varchar(50),url varchar(200))"
|
||||
self.dbHelper.createTable(sql)
|
||||
|
||||
def testInsert(self):
|
||||
sql="insert into testtable(name,url) values(%s,%s)"
|
||||
params=("test","test")
|
||||
self.dbHelper.insert(sql,*params)
|
||||
def testUpdate(self):
|
||||
sql="update testtable set name=%s,url=%s where id=%s"
|
||||
params=("update","update","1")
|
||||
self.dbHelper.update(sql,*params)
|
||||
|
||||
def testDelete(self):
|
||||
sql="delete from testtable where id=%s"
|
||||
params=("1")
|
||||
self.dbHelper.delete(sql,*params)
|
||||
|
||||
if __name__=="__main__":
|
||||
testDBHelper=TestDBHelper()
|
||||
#testDBHelper.testCreateDatebase() #
|
||||
#testDBHelper.testCreateTable() #
|
||||
#testDBHelper.testInsert() #
|
||||
#testDBHelper.testUpdate() #
|
||||
#testDBHelper.testDelete() #
|
||||
@@ -0,0 +1,27 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/items.html
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
class LiepinspdItem(scrapy.Item):
|
||||
# define the fields for your item here like:
|
||||
as_of_date = scrapy.Field()
|
||||
ticker = scrapy.Field()
|
||||
company_name = scrapy.Field()
|
||||
stage = scrapy.Field()
|
||||
size = scrapy.Field()
|
||||
city = scrapy.Field()
|
||||
industry = scrapy.Field()
|
||||
comp_clearfix = scrapy.Field()
|
||||
rate_num = scrapy.Field()
|
||||
job_count = scrapy.Field()
|
||||
registered_capital = scrapy.Field()
|
||||
|
||||
spider_time = scrapy.Field()
|
||||
origin_site = scrapy.Field()
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your spider middleware
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
from scrapy import signals
|
||||
import scrapy
|
||||
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
|
||||
import random
|
||||
|
||||
class LiepinspdSpiderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the spider middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_spider_input(self, response, spider):
|
||||
# Called for each response that goes through the spider
|
||||
# middleware and into the spider.
|
||||
|
||||
# Should return None or raise an exception.
|
||||
return None
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
# Called with the results returned from the Spider, after
|
||||
# it has processed the response.
|
||||
|
||||
# Must return an iterable of Request, dict or Item objects.
|
||||
for i in result:
|
||||
yield i
|
||||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
# Called when a spider or process_spider_input() method
|
||||
# (from other spider middleware) raises an exception.
|
||||
|
||||
# Should return either None or an iterable of Response, dict
|
||||
# or Item objects.
|
||||
pass
|
||||
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
# Called with the start requests of the spider, and works
|
||||
# similarly to the process_spider_output() method, except
|
||||
# that it doesn’t have a response associated.
|
||||
|
||||
# Must return only requests (not items).
|
||||
for r in start_requests:
|
||||
yield r
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class LiepinspdDownloaderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the downloader middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_request(self, request, spider):
|
||||
# Called for each request that goes through the downloader
|
||||
# middleware.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this request
|
||||
# - or return a Response object
|
||||
# - or return a Request object
|
||||
# - or raise IgnoreRequest: process_exception() methods of
|
||||
# installed downloader middleware will be called
|
||||
return None
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
# Called with the response returned from the downloader.
|
||||
|
||||
# Must either;
|
||||
# - return a Response object
|
||||
# - return a Request object
|
||||
# - or raise IgnoreRequest
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
# Called when a download handler or a process_request()
|
||||
# (from other downloader middleware) raises an exception.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this exception
|
||||
# - return a Response object: stops process_exception() chain
|
||||
# - return a Request object: stops process_exception() chain
|
||||
pass
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class MyUserAgentMiddleware(UserAgentMiddleware):
|
||||
'''
|
||||
设置User-Agent
|
||||
'''
|
||||
|
||||
def __init__(self, user_agent):
|
||||
self.user_agent = user_agent
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(
|
||||
user_agent=crawler.settings.get('USER_AGENTS')
|
||||
)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
agent = random.choice(self.user_agent)
|
||||
request.headers['User-Agent'] = agent
|
||||
@@ -0,0 +1,89 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
from twisted.enterprise import adbapi
|
||||
import pymysql
|
||||
import pymysql.cursors
|
||||
|
||||
# class LiepinspdPipeline(object):
|
||||
# def __init__(self, dbpool):
|
||||
# self.dbpool = dbpool
|
||||
#
|
||||
# @classmethod
|
||||
# def from_settings(cls, settings): # 函数名固定，会被scrapy调用，直接可用settings的值
|
||||
# """
|
||||
# 数据库建立连接
|
||||
# :param settings: 配置参数
|
||||
# :return: 实例化参数
|
||||
# """
|
||||
#
|
||||
# adbparams = dict(
|
||||
# host=settings['MYSQL_HOST'],
|
||||
# db=settings['MYSQL_DBNAME'],
|
||||
# user=settings['MYSQL_USER'],
|
||||
# password=settings['MYSQL_PASSWORD'],
|
||||
# port = settings['MYSQL_PORT'],
|
||||
# cursorclass=pymysql.cursors.DictCursor # 指定cursor类型
|
||||
# )
|
||||
# # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
|
||||
# dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
|
||||
# # 返回实例化参数
|
||||
# return cls(dbpool)
|
||||
#
|
||||
# def process_item(self, item, spider):
|
||||
# """
|
||||
# 使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
|
||||
# """
|
||||
# query = self.dbpool.runInteraction(self.do_insert, item) # 指定操作方法和操作数据
|
||||
# # 添加异常处理
|
||||
# query.addCallback(self.handle_error) # 处理异常
|
||||
#
|
||||
# def do_insert(self, cursor, item):
|
||||
# # 对数据库进行插入操作，并不需要commit，twisted会自动commit
|
||||
#
|
||||
# insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industy,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
|
||||
# cursor.execute(insert_sql,
|
||||
# (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
|
||||
# str(item['size']), str(item['city']), str(item['industy']), str(item['comp_clearfix']),
|
||||
# int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),item['spider_time'],item['origin_site'],))
|
||||
# def handle_error(self, failure):
|
||||
# if failure:
|
||||
# # 打印错误信息
|
||||
# print(failure)
|
||||
|
||||
|
||||
import pymysql
|
||||
|
||||
|
||||
class LiepinspdPipeline(object):
|
||||
"""
|
||||
同步操作
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# 建立连接
|
||||
self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data') # 有中文要存入数据库的话要加charset='utf8'
|
||||
# 创建游标
|
||||
self.cursor = self.conn.cursor()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
# sql语句
|
||||
insert_sql = """
|
||||
insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industry,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
||||
"""
|
||||
# 执行插入数据到数据库操作
|
||||
self.cursor.execute(insert_sql,
|
||||
(item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
|
||||
str(item['size']), str(item['city']), str(item['industry']), str(item['comp_clearfix']),
|
||||
int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),
|
||||
item['spider_time'], item['origin_site'],))
|
||||
# 提交，不进行提交无法保存到数据库
|
||||
self.conn.commit()
|
||||
|
||||
def close_spider(self, spider):
|
||||
# 关闭游标和连接
|
||||
self.cursor.close()
|
||||
self.conn.close()
|
||||
@@ -0,0 +1,211 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Scrapy settings for liepinSpd project
|
||||
#
|
||||
# For simplicity, this file contains only settings considered important or
|
||||
# commonly used. You can find more settings consulting the documentation:
|
||||
#
|
||||
# https://doc.scrapy.org/en/latest/topics/settings.html
|
||||
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
COMPANYLIST=['7894126', '7941798', '5464493', '8280653', '8657147', '5696000', '6918711', '8801813', '7909112', '929719', '8635277', '9208490', '9427534', '7873563', '869131', '1983198', '8521820', '8441886', '9425884', '8269623', '8143143', '8144649', '8571478', '8646314', '9086358', '8361354', '8090600', '9652027', '9662729', '8029798', '8024700', '9274661', '8614537', '1852098', '845611', '7910884', '1947829', '6657987', '8463020', '8130349', '8323671', '723421', '1573297', '9582057', '1866404', '1074696', '8586065', '4811624', '857922', '7975388', '7931578', '6615613', '8243943', '682357', '8916773', '1050201', '950043', '7939262', '1730543', '9469426', '7883086', '8628525', '7868218', '8096323', '7862738', '7023768', '8862767', '9538671', '7953390', '515361', '2104592', '993518', '8212985', '1766564', '892388', '8646248', '9857531', '1043007', '8042835', '8980779', '571837', '7862722', '7935093', '8130825', '9111311', '8051561', '9107424', '856576', '7862125', '7947928', '854827', '4209085', '859352', '7931740', '7939262', '548548', '7916182', '8354065', '9740398', '8155722', '2331894', '884195', '9651734', '8534019', '7855573', '9617356', '886895', '2431058', '1939058', '8246296', '9145034', '8161625', '4450360', '540933', '4817469']
|
||||
|
||||
DEFAULT_REQUEST_HEADERS = {
|
||||
'Connection': 'keep-alive',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9'
|
||||
}
|
||||
|
||||
BOT_NAME = 'liepinSpd'
|
||||
|
||||
MYSQL_HOST = 'rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com'
|
||||
MYSQL_DBNAME = 'special_data'
|
||||
MYSQL_USER = 'cn_ainvest_db'
|
||||
MYSQL_PASSWD = 'cn_ainvest_sd3a1'
|
||||
MYSQL_PORT = 3306
|
||||
|
||||
SPIDER_MODULES = ['liepinSpd.spiders']
|
||||
NEWSPIDER_MODULE = 'liepinSpd.spiders'
|
||||
|
||||
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
#USER_AGENT = 'liepinSpd (+http://www.yourdomain.com)'
|
||||
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = False
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
|
||||
"Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
|
||||
"Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0"
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
|
||||
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
|
||||
"Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
|
||||
"Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
|
||||
"Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
|
||||
'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
|
||||
'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
|
||||
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
|
||||
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
|
||||
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
|
||||
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
|
||||
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
|
||||
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
|
||||
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
|
||||
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
|
||||
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||
"HTC_Dream Mozilla/5.0 (Linux; U; Android 1.5; en-ca; Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; en-us; sdk Build/CUPCAKE) AppleWebkit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; en-us; htc_bahamas Build/CRB17) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.1-update1; de-de; HTC Desire 1.19.161.5 Build/ERE27) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; de-ch; HTC Hero Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.1; en-us; HTC Legend Build/cupcake) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; de-de; HTC Magic Build/PLAT-RC33) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1 FirePHP/0.3",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.6; en-us; HTC_TATTOO_A3288 Build/DRC79) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; en-us; T-Mobile G1 Build/CRB43) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari 525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.5; en-gb; T-Mobile_G2_Touch Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Droid Build/FRG22D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.0; en-us; Milestone Build/ SHOLS_U2_01.03.1) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.0.1; de-de; Milestone Build/SHOLS_U2_01.14.0) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
|
||||
"Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522 (KHTML, like Gecko) Safari/419.3",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.1; en-gb; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
|
||||
"Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.6; es-es; SonyEricssonX10i Build/R1FA016) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
"Mozilla/5.0 (Linux; U; Android 1.6; en-us; SonyEricssonX10i Build/R1AA056) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1",
|
||||
]
|
||||
|
||||
# Configure maximum concurrent requests performed by Scrapy (default: 16)
|
||||
#CONCURRENT_REQUESTS = 32
|
||||
|
||||
# Configure a delay for requests for the same website (default: 0)
|
||||
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
|
||||
# See also autothrottle settings and docs
|
||||
DOWNLOAD_DELAY = 3
|
||||
# The download delay setting will honor only one of:
|
||||
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
||||
#CONCURRENT_REQUESTS_PER_IP = 16
|
||||
|
||||
# Disable cookies (enabled by default)
|
||||
#COOKIES_ENABLED = False
|
||||
|
||||
# Disable Telnet Console (enabled by default)
|
||||
#TELNETCONSOLE_ENABLED = False
|
||||
|
||||
# Override the default request headers:
|
||||
|
||||
# Enable or disable spider middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
#SPIDER_MIDDLEWARES = {
|
||||
# 'liepinSpd.middlewares.LiepinspdSpiderMiddleware': 543,
|
||||
#}
|
||||
|
||||
# Enable or disable downloader middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
# 'liepinSpd.middlewares.LiepinspdDownloaderMiddleware': 543,
|
||||
'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
|
||||
'liepinSpd.middlewares.MyUserAgentMiddleware': 400,
|
||||
}
|
||||
|
||||
# Enable or disable extensions
|
||||
# See https://doc.scrapy.org/en/latest/topics/extensions.html
|
||||
#EXTENSIONS = {
|
||||
# 'scrapy.extensions.telnet.TelnetConsole': None,
|
||||
#}
|
||||
|
||||
# Configure item pipelines
|
||||
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
ITEM_PIPELINES = {
|
||||
'liepinSpd.pipelines.LiepinspdPipeline': 300,
|
||||
}
|
||||
|
||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
|
||||
#AUTOTHROTTLE_ENABLED = True
|
||||
# The initial download delay
|
||||
#AUTOTHROTTLE_START_DELAY = 5
|
||||
# The maximum download delay to be set in case of high latencies
|
||||
#AUTOTHROTTLE_MAX_DELAY = 60
|
||||
# The average number of requests Scrapy should be sending in parallel to
|
||||
# each remote server
|
||||
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||
# Enable showing throttling stats for every response received:
|
||||
#AUTOTHROTTLE_DEBUG = False
|
||||
|
||||
# Enable and configure HTTP caching (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||
#HTTPCACHE_ENABLED = True
|
||||
#HTTPCACHE_EXPIRATION_SECS = 0
|
||||
#HTTPCACHE_DIR = 'httpcache'
|
||||
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
@@ -0,0 +1,4 @@
|
||||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# Please refer to the documentation for information on how to create and manage
|
||||
# your spiders.
|
||||
@@ -0,0 +1,90 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import scrapy
|
||||
import re
|
||||
from datetime import datetime
|
||||
import pandas as pd
|
||||
import time
|
||||
|
||||
from liepinSpd.items import LiepinspdItem
|
||||
|
||||
|
||||
class LiepinSpdier(scrapy.Spider):
|
||||
name = 'liepin'
|
||||
data = pd.read_csv('G:\workspace\y2019m02\company500.csv', encoding='utf-8')
|
||||
companylist=data['股票简称']
|
||||
start_urls = []
|
||||
for company in companylist:
|
||||
start_urls.append(f'https://www.liepin.com/zhaopin/?key={company}')
|
||||
|
||||
# 公司主要基本信息
|
||||
def parse(self, response):
|
||||
# company = response.meta['company']
|
||||
text = response.text
|
||||
# print(text)
|
||||
# 抓取公司基本信息
|
||||
# try:
|
||||
company_name = response.xpath('//div[@class="name-and-welfare"]//h1/text()')[0].extract()
|
||||
# print(company_name)
|
||||
comp_sum_tag = response.xpath('//div[@class="comp-summary-tag"]/a/text()').extract()
|
||||
# 好几个
|
||||
stage=comp_sum_tag[0]
|
||||
# print(stage)
|
||||
size=comp_sum_tag[1]
|
||||
# print(size)
|
||||
city=comp_sum_tag[2]
|
||||
# print(city)
|
||||
industry=comp_sum_tag[3]
|
||||
# print(industy)
|
||||
#公司标签,list
|
||||
comp_clearfix = str(response.xpath('//ul[@class="comp-tag-list clearfix"]//span/text()').extract())
|
||||
# print(comp_clearfix)
|
||||
#简历处理率 *%转化为float
|
||||
rate_num = response.xpath('//p[@class="rate-num"]//span/text()')[0].extract()
|
||||
rate_num=int(rate_num)/100
|
||||
# print(rate_num)
|
||||
|
||||
job_count = int(re.search(r'<small data-selector="total">. 共([0-9]+) 个', text).group(1))
|
||||
# print(job_count)
|
||||
#注册资本(万元)
|
||||
if '注册资本' in text and '万元人民币' in text:
|
||||
registered_capital = float(re.search(r'<li>注册资本：(.*?)万元人民币</li>', text).group(1))
|
||||
else:
|
||||
registered_capital =0.0
|
||||
# print(registered_capital)
|
||||
origin_site=re.search(r'"wapUrl":"(.*?)",', text).group(1)
|
||||
item = LiepinspdItem()
|
||||
# 匹配股票代码,判断如果股票简称全部在公司名内,则匹配股票代码
|
||||
data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
|
||||
try:
|
||||
for i in range(len(data)):
|
||||
n = 0
|
||||
for j in data.loc[i, '股票简称']:
|
||||
if j in company_name:
|
||||
n += 1
|
||||
if n == len(data.loc[i, '股票简称']):
|
||||
item['ticker'] = data.loc[i, '股票代码']
|
||||
# print(n, item['ticker'], company_name)
|
||||
# else:
|
||||
# item['ticker'] ='未匹配'
|
||||
except BaseException as e:
|
||||
print('ticker匹配错误')
|
||||
|
||||
item['as_of_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
item['company_name'] = company_name
|
||||
item['stage'] = stage
|
||||
item['size'] = size
|
||||
item['city'] = city
|
||||
item['industry'] = industry
|
||||
item['comp_clearfix'] = comp_clearfix
|
||||
item['rate_num'] = rate_num
|
||||
item['job_count'] = job_count
|
||||
item['registered_capital'] = registered_capital
|
||||
item['spider_time'] = datetime.strptime(str(datetime.now())[:10], '%Y-%m-%d').date()
|
||||
item['origin_site'] = origin_site
|
||||
|
||||
yield item
|
||||
# except BaseException as e:
|
||||
# print('error and pass')
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# 获取settings.py模块的设置
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
from liepinSpd.spiders.lpspider import LiepinSpdier
|
||||
|
||||
settings = get_project_settings()
|
||||
process = CrawlerProcess(settings=settings)
|
||||
|
||||
# 可以添加多个spider类
|
||||
process.crawl(LiepinSpdier)
|
||||
|
||||
# 启动爬虫，会阻塞，直到爬取完成
|
||||
process.start()
|
||||
@@ -0,0 +1,11 @@
|
||||
# Automatically created by: scrapy startproject
|
||||
#
|
||||
# For more information about the [deploy] section see:
|
||||
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
|
||||
[settings]
|
||||
default = liepinSpd.settings
|
||||
|
||||
[deploy]
|
||||
#url = http://localhost:6800/
|
||||
project = liepinSpd
|
||||
@@ -0,0 +1,24 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/items.html
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
class LiepinspecialcomItem(scrapy.Item):
|
||||
# define the fields for your item here like:
|
||||
# name = scrapy.Field()
|
||||
as_of_date = scrapy.Field()
|
||||
ticker = scrapy.Field()
|
||||
company_name = scrapy.Field()
|
||||
# stage = scrapy.Field()
|
||||
size = scrapy.Field()
|
||||
city = scrapy.Field()
|
||||
industry = scrapy.Field()
|
||||
# comp_clearfix = scrapy.Field()
|
||||
# job_count = scrapy.Field()
|
||||
# rate_num = scrapy.Field()
|
||||
# registered_capital = scrapy.Field()
|
||||
@@ -0,0 +1,126 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your spider middleware
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy import signals
|
||||
import scrapy
|
||||
import random
|
||||
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
|
||||
|
||||
|
||||
class LiepinspecialcomSpiderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the spider middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_spider_input(self, response, spider):
|
||||
# Called for each response that goes through the spider
|
||||
# middleware and into the spider.
|
||||
|
||||
# Should return None or raise an exception.
|
||||
return None
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
# Called with the results returned from the Spider, after
|
||||
# it has processed the response.
|
||||
|
||||
# Must return an iterable of Request, dict or Item objects.
|
||||
for i in result:
|
||||
yield i
|
||||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
# Called when a spider or process_spider_input() method
|
||||
# (from other spider middleware) raises an exception.
|
||||
|
||||
# Should return either None or an iterable of Response, dict
|
||||
# or Item objects.
|
||||
pass
|
||||
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
# Called with the start requests of the spider, and works
|
||||
# similarly to the process_spider_output() method, except
|
||||
# that it doesn’t have a response associated.
|
||||
|
||||
# Must return only requests (not items).
|
||||
for r in start_requests:
|
||||
yield r
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class LiepinspecialcomDownloaderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the downloader middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_request(self, request, spider):
|
||||
# Called for each request that goes through the downloader
|
||||
# middleware.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this request
|
||||
# - or return a Response object
|
||||
# - or return a Request object
|
||||
# - or raise IgnoreRequest: process_exception() methods of
|
||||
# installed downloader middleware will be called
|
||||
return None
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
# Called with the response returned from the downloader.
|
||||
|
||||
# Must either;
|
||||
# - return a Response object
|
||||
# - return a Request object
|
||||
# - or raise IgnoreRequest
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
# Called when a download handler or a process_request()
|
||||
# (from other downloader middleware) raises an exception.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this exception
|
||||
# - return a Response object: stops process_exception() chain
|
||||
# - return a Request object: stops process_exception() chain
|
||||
pass
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class MyUserAgentMiddleware(UserAgentMiddleware):
|
||||
'''
|
||||
设置User-Agent
|
||||
'''
|
||||
|
||||
def __init__(self, user_agent):
|
||||
self.user_agent = user_agent
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(
|
||||
user_agent=crawler.settings.get('USER_AGENTS')
|
||||
)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
agent = random.choice(self.user_agent)
|
||||
request.headers['User-Agent'] = agent
|
||||
@@ -0,0 +1,132 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
|
||||
from twisted.enterprise import adbapi
|
||||
import pymysql
|
||||
import pymysql.cursors
|
||||
|
||||
# class LiepinspdPipeline(object):
|
||||
# def __init__(self, dbpool):
|
||||
# self.dbpool = dbpool
|
||||
#
|
||||
# @classmethod
|
||||
# def from_settings(cls, settings): # 函数名固定，会被scrapy调用，直接可用settings的值
|
||||
# """
|
||||
# 数据库建立连接
|
||||
# :param settings: 配置参数
|
||||
# :return: 实例化参数
|
||||
# """
|
||||
#
|
||||
# adbparams = dict(
|
||||
# host=settings['MYSQL_HOST'],
|
||||
# db=settings['MYSQL_DBNAME'],
|
||||
# user=settings['MYSQL_USER'],
|
||||
# password=settings['MYSQL_PASSWORD'],
|
||||
# port = settings['MYSQL_PORT'],
|
||||
# cursorclass=pymysql.cursors.DictCursor # 指定cursor类型
|
||||
# )
|
||||
# # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
|
||||
# dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
|
||||
# # 返回实例化参数
|
||||
# return cls(dbpool)
|
||||
#
|
||||
# def process_item(self, item, spider):
|
||||
# """
|
||||
# 使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
|
||||
# """
|
||||
# query = self.dbpool.runInteraction(self.do_insert, item) # 指定操作方法和操作数据
|
||||
# # 添加异常处理
|
||||
# query.addCallback(self.handle_error) # 处理异常
|
||||
#
|
||||
# def do_insert(self, cursor, item):
|
||||
# # 对数据库进行插入操作，并不需要commit，twisted会自动commit
|
||||
#
|
||||
# insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,stage,`size`,city,industy,comp_clearfix,job_count,rate_num,registered_capital,spider_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
|
||||
# cursor.execute(insert_sql,
|
||||
# (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['stage']),
|
||||
# str(item['size']), str(item['city']), str(item['industy']), str(item['comp_clearfix']),
|
||||
# int(item['job_count']), float(item['rate_num']), float(item['registered_capital']),item['spider_time'],item['origin_site'],))
|
||||
# def handle_error(self, failure):
|
||||
# if failure:
|
||||
# # 打印错误信息
|
||||
# print(failure)
|
||||
|
||||
|
||||
import pymysql
|
||||
|
||||
|
||||
class LiepinspecialcomPipeline(object):
|
||||
"""
|
||||
同步操作
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# 建立连接
|
||||
self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data') # 有中文要存入数据库的话要加charset='utf8'
|
||||
# 创建游标
|
||||
self.cursor = self.conn.cursor()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
# sql语句
|
||||
insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,`size`,city,industry) VALUES(%s,%s,%s,%s,%s,%s)"
|
||||
# 执行插入数据到数据库操作
|
||||
self.cursor.execute(insert_sql,
|
||||
(item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['size']),
|
||||
str(item['city']),str(item['industry'])))
|
||||
# 提交，不进行提交无法保存到数据库
|
||||
self.conn.commit()
|
||||
|
||||
def close_spider(self, spider):
|
||||
# 关闭游标和连接
|
||||
self.cursor.close()
|
||||
self.conn.close()
|
||||
|
||||
|
||||
|
||||
# class LiepinspecialcomPipeline(object):
|
||||
# def __init__(self, dbpool):
|
||||
# self.dbpool = dbpool
|
||||
#
|
||||
# @classmethod
|
||||
# def from_settings(cls, settings): # 函数名固定，会被scrapy调用，直接可用settings的值
|
||||
# """
|
||||
# 数据库建立连接
|
||||
# :param settings: 配置参数
|
||||
# :return: 实例化参数
|
||||
# """
|
||||
#
|
||||
# adbparams = dict(
|
||||
# host=settings['MYSQL_HOST'],
|
||||
# db=settings['MYSQL_DBNAME'],
|
||||
# user=settings['MYSQL_USER'],
|
||||
# password=settings['MYSQL_PASSWORD'],
|
||||
# cursorclass=pymysql.cursors.DictCursor # 指定cursor类型
|
||||
# )
|
||||
# # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
|
||||
# dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
|
||||
# # 返回实例化参数
|
||||
# return cls(dbpool)
|
||||
#
|
||||
# def process_item(self, item, spider):
|
||||
# """
|
||||
# 使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
|
||||
# """
|
||||
# query = self.dbpool.runInteraction(self.do_insert, item) # 指定操作方法和操作数据
|
||||
# # 添加异常处理
|
||||
# query.addCallback(self.handle_error) # 处理异常
|
||||
#
|
||||
# def do_insert(self, cursor, item):
|
||||
# # 对数据库进行插入操作，并不需要commit，twisted会自动commit
|
||||
# insert_sql = "insert into company_base_info(as_of_date,ticker,company_name,`size`,city,industry) VALUES(%s,%s,%s,%s,%s,%s)"
|
||||
# cursor.execute(insert_sql, (
|
||||
# item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['size']), str(item['city']),
|
||||
# str(item['industry'])))
|
||||
#
|
||||
# def handle_error(self, failure):
|
||||
# if failure:
|
||||
# # 打印错误信息
|
||||
# print(failure)
|
||||
@@ -0,0 +1,175 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Scrapy settings for liepinSpecialCom project
|
||||
#
|
||||
# For simplicity, this file contains only settings considered important or
|
||||
# commonly used. You can find more settings consulting the documentation:
|
||||
#
|
||||
# https://doc.scrapy.org/en/latest/topics/settings.html
|
||||
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
BOT_NAME = 'liepinSpecialCom'
|
||||
|
||||
MYSQL_HOST = 'localhost'
|
||||
MYSQL_DBNAME = 'day0123'
|
||||
MYSQL_USER = 'root'
|
||||
MYSQL_PASSWD = '123'
|
||||
|
||||
SPIDER_MODULES = ['liepinSpecialCom.spiders']
|
||||
NEWSPIDER_MODULE = 'liepinSpecialCom.spiders'
|
||||
|
||||
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
#USER_AGENT = 'liepinSpecialCom (+http://www.yourdomain.com)'
|
||||
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = False
|
||||
|
||||
# Configure maximum concurrent requests performed by Scrapy (default: 16)
|
||||
#CONCURRENT_REQUESTS = 32
|
||||
|
||||
# Configure a delay for requests for the same website (default: 0)
|
||||
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
|
||||
# See also autothrottle settings and docs
|
||||
DOWNLOAD_DELAY = 3
|
||||
# The download delay setting will honor only one of:
|
||||
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
||||
#CONCURRENT_REQUESTS_PER_IP = 16
|
||||
|
||||
# Disable cookies (enabled by default)
|
||||
#COOKIES_ENABLED = False
|
||||
|
||||
# Disable Telnet Console (enabled by default)
|
||||
#TELNETCONSOLE_ENABLED = False
|
||||
|
||||
# Override the default request headers:
|
||||
DEFAULT_REQUEST_HEADERS = {
|
||||
'Connection': 'keep-alive',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9'
|
||||
}
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
|
||||
"Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
|
||||
"Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0"
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
|
||||
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
|
||||
"Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
|
||||
"Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
|
||||
"Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
|
||||
'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
|
||||
'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
|
||||
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
|
||||
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
|
||||
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)'
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
|
||||
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
|
||||
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
|
||||
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
|
||||
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
|
||||
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
|
||||
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||
]
|
||||
|
||||
# Enable or disable spider middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
#SPIDER_MIDDLEWARES = {
|
||||
# 'liepinSpecialCom.middlewares.LiepinspecialcomSpiderMiddleware': 543,
|
||||
#}
|
||||
|
||||
# Enable or disable downloader middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
# 'liepinSpecialCom.middlewares.LiepinspecialcomDownloaderMiddleware': 543,
|
||||
'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
|
||||
'liepinSpecialCom.middlewares.MyUserAgentMiddleware': 400,
|
||||
}
|
||||
|
||||
# Enable or disable extensions
|
||||
# See https://doc.scrapy.org/en/latest/topics/extensions.html
|
||||
#EXTENSIONS = {
|
||||
# 'scrapy.extensions.telnet.TelnetConsole': None,
|
||||
#}
|
||||
|
||||
# Configure item pipelines
|
||||
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
ITEM_PIPELINES = {
|
||||
'liepinSpecialCom.pipelines.LiepinspecialcomPipeline': 300,
|
||||
}
|
||||
|
||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
|
||||
#AUTOTHROTTLE_ENABLED = True
|
||||
# The initial download delay
|
||||
#AUTOTHROTTLE_START_DELAY = 5
|
||||
# The maximum download delay to be set in case of high latencies
|
||||
#AUTOTHROTTLE_MAX_DELAY = 60
|
||||
# The average number of requests Scrapy should be sending in parallel to
|
||||
# each remote server
|
||||
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||
# Enable showing throttling stats for every response received:
|
||||
#AUTOTHROTTLE_DEBUG = False
|
||||
|
||||
# Enable and configure HTTP caching (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||
#HTTPCACHE_ENABLED = True
|
||||
#HTTPCACHE_EXPIRATION_SECS = 0
|
||||
#HTTPCACHE_DIR = 'httpcache'
|
||||
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
@@ -0,0 +1,4 @@
|
||||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# Please refer to the documentation for information on how to create and manage
|
||||
# your spiders.
|
||||
@@ -0,0 +1,89 @@
|
||||
import scrapy
|
||||
import re
|
||||
from datetime import datetime
|
||||
import pandas as pd
|
||||
import time
|
||||
|
||||
from liepinSpd.items import LiepinspdItem
|
||||
|
||||
|
||||
class LiepinSpdier(scrapy.Spider):
|
||||
name = 'liepin'
|
||||
start_urls = ['https://www.liepin.com/job/1917579081.shtml?d_sfrom=search_comp&d_ckId=d112af69ab58e7da8305520f55b31904&d_curPage=0&d_pageSize=15&d_headId=d112af69ab58e7da8305520f55b31904&d_posi=0',
|
||||
'https://www.liepin.com/job/1917549017.shtml?d_sfrom=search_comp&d_ckId=1bef90aa98c2e8da734552c320527ac0&d_curPage=0&d_pageSize=15&d_headId=1bef90aa98c2e8da734552c320527ac0&d_posi=0',
|
||||
'https://www.liepin.com/job/1917543155.shtml',
|
||||
'https://www.liepin.com/job/1917491571.shtml?d_sfrom=search_comp&d_ckId=5874ecde43eb4bd20e75fecb2709bf85&d_curPage=0&d_pageSize=15&d_headId=5874ecde43eb4bd20e75fecb2709bf85&d_posi=0',
|
||||
'https://www.liepin.com/job/1917505785.shtml?d_sfrom=search_comp&d_ckId=fe82f0f79cda01b1dd4c140ced26087c&d_curPage=0&d_pageSize=15&d_headId=fe82f0f79cda01b1dd4c140ced26087c&d_posi=0',
|
||||
'https://www.liepin.com/job/1916439263.shtml?d_sfrom=search_comp&d_ckId=d3f4428da37a0cd17a6235cb4a027f1e&d_curPage=0&d_pageSize=15&d_headId=d3f4428da37a0cd17a6235cb4a027f1e&d_posi=0',
|
||||
'https://www.liepin.com/job/1911157736.shtml?d_sfrom=search_comp&d_ckId=2cf44398e8273003087d5148e113ef8f&d_curPage=0&d_pageSize=15&d_headId=2cf44398e8273003087d5148e113ef8f&d_posi=0',
|
||||
'https://www.liepin.com/job/1917470663.shtml?d_sfrom=search_comp&d_ckId=9087e4fc55d61d200606fb906999f728&d_curPage=0&d_pageSize=15&d_headId=9087e4fc55d61d200606fb906999f728&d_posi=0',
|
||||
'https://www.liepin.com/job/1917533673.shtml?d_sfrom=search_comp&d_ckId=98408645fba7219d4d7f17f2714c96f0&d_curPage=0&d_pageSize=15&d_headId=98408645fba7219d4d7f17f2714c96f0&d_posi=0',
|
||||
'https://www.liepin.com/job/1917306593.shtml?d_sfrom=search_comp&d_ckId=85f632646e2b1ad7c06f436e25fd674d&d_curPage=0&d_pageSize=15&d_headId=85f632646e2b1ad7c06f436e25fd674d&d_posi=0',
|
||||
'https://www.liepin.com/job/199929552.shtml'
|
||||
]
|
||||
|
||||
# 公司主要基本信息
|
||||
def parse(self, response):
|
||||
text = response.text
|
||||
# print(text)
|
||||
# 抓取公司基本信息
|
||||
# try:
|
||||
company_name = response.xpath('//div[@class="about-position"]//a/text()')[0].extract()
|
||||
# print(company_name)
|
||||
# comp_sum_tag = response.xpath('//div[@class="comp-summary-tag"]/a/text()').extract()
|
||||
# 好几个
|
||||
# stage = comp_sum_tag[0]
|
||||
# print(stage)
|
||||
size = re.search(r'公司规模：(.*?)人',text).group(1)
|
||||
# print(size)
|
||||
city = re.search(r'公司地址：(.*?)<',text).group(1)
|
||||
# print(city)
|
||||
industry = re.search(r'行业.*?>(.*?)<',text).group(1)
|
||||
# print(industy)
|
||||
# 公司标签,list
|
||||
# comp_clearfix = str(response.xpath('//ul[@class="comp-tag-list clearfix"]//span/text()').extract())
|
||||
# print(comp_clearfix)
|
||||
# 简历处理率 *%转化为float
|
||||
# rate_num = response.xpath('//p[@class="rate-num"]//span/text()')[0].extract()
|
||||
# rate_num = int(rate_num) / 100
|
||||
# print(rate_num)
|
||||
|
||||
# job_count = int(re.search(r'<small data-selector="total">. 共([0-9]+) 个', text).group(1))
|
||||
# print(job_count)
|
||||
# 注册资本(万元)
|
||||
# registered_capital = float(re.search(r'<li>注册资本：(.*?)万元人民币</li>', text).group(1))
|
||||
# print(registered_capital)
|
||||
|
||||
as_of_date = datetime.now() # 最后确认一下格式是否正确
|
||||
|
||||
item = LiepinspdItem()
|
||||
# 匹配股票代码,判断如果股票简称全部在公司名内,则匹配股票代码
|
||||
data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
|
||||
try:
|
||||
for i in range(len(data)):
|
||||
n = 0
|
||||
for j in data.loc[i, '股票简称']:
|
||||
if j in company_name:
|
||||
n += 1
|
||||
if n >= len(data.loc[i, '股票简称'])-1:
|
||||
item['ticker'] = data.loc[i, '股票代码']
|
||||
print(n, item['ticker'], company_name)
|
||||
except BaseException as e:
|
||||
item['ticker'] ='None'
|
||||
print('ticker匹配错误')
|
||||
|
||||
item['as_of_date'] = as_of_date
|
||||
item['company_name'] = company_name
|
||||
# item['stage'] = stage
|
||||
item['size'] = size
|
||||
item['city'] = city
|
||||
item['industry'] = industry
|
||||
# item['comp_clearfix'] = comp_clearfix
|
||||
# item['rate_num'] = rate_num
|
||||
# item['job_count'] = job_count
|
||||
# item['registered_capital'] = registered_capital
|
||||
# time.sleep(2)
|
||||
|
||||
yield item
|
||||
# except BaseException as e:
|
||||
# print('error and pass')
|
||||
@@ -0,0 +1,17 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# 获取settings.py模块的设置
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
from liepinSpecialCom.spiders.lpspecialcom import LiepinSpdier
|
||||
|
||||
settings = get_project_settings()
|
||||
process = CrawlerProcess(settings=settings)
|
||||
|
||||
# 可以添加多个spider类
|
||||
process.crawl(LiepinSpdier)
|
||||
|
||||
# 启动爬虫，会阻塞，直到爬取完成
|
||||
process.start()
|
||||
@@ -0,0 +1,11 @@
|
||||
# Automatically created by: scrapy startproject
|
||||
#
|
||||
# For more information about the [deploy] section see:
|
||||
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
|
||||
[settings]
|
||||
default = liepinSpecialCom.settings
|
||||
|
||||
[deploy]
|
||||
#url = http://localhost:6800/
|
||||
project = liepinSpecialCom
|
||||
@@ -0,0 +1,23 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/items.html
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
class LiepinspecialcomjobItem(scrapy.Item):
|
||||
# define the fields for your item here like:
|
||||
# name = scrapy.Field()
|
||||
as_of_date = scrapy.Field()
|
||||
ticker = scrapy.Field()
|
||||
company_name = scrapy.Field()
|
||||
job_name = scrapy.Field()
|
||||
salary = scrapy.Field()
|
||||
city = scrapy.Field()
|
||||
work_year = scrapy.Field()
|
||||
pub_time = scrapy.Field()
|
||||
education = scrapy.Field()
|
||||
origin_site = scrapy.Field()
|
||||
@@ -0,0 +1,159 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your spider middleware
|
||||
#
|
||||
# See documentation in:
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
import random
|
||||
import time
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
|
||||
|
||||
class LiepinspecialcomjobSpiderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the spider middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_spider_input(self, response, spider):
|
||||
# Called for each response that goes through the spider
|
||||
# middleware and into the spider.
|
||||
|
||||
# Should return None or raise an exception.
|
||||
return None
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
# Called with the results returned from the Spider, after
|
||||
# it has processed the response.
|
||||
|
||||
# Must return an iterable of Request, dict or Item objects.
|
||||
for i in result:
|
||||
yield i
|
||||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
# Called when a spider or process_spider_input() method
|
||||
# (from other spider middleware) raises an exception.
|
||||
|
||||
# Should return either None or an iterable of Response, dict
|
||||
# or Item objects.
|
||||
pass
|
||||
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
# Called with the start requests of the spider, and works
|
||||
# similarly to the process_spider_output() method, except
|
||||
# that it doesn’t have a response associated.
|
||||
|
||||
# Must return only requests (not items).
|
||||
for r in start_requests:
|
||||
yield r
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class LiepinspecialcomjobDownloaderMiddleware(object):
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
# scrapy acts as if the downloader middleware does not modify the
|
||||
# passed objects.
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
# This method is used by Scrapy to create your spiders.
|
||||
s = cls()
|
||||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_request(self, request, spider):
|
||||
# Called for each request that goes through the downloader
|
||||
# middleware.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this request
|
||||
# - or return a Response object
|
||||
# - or return a Request object
|
||||
# - or raise IgnoreRequest: process_exception() methods of
|
||||
# installed downloader middleware will be called
|
||||
return None
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
# Called with the response returned from the downloader.
|
||||
|
||||
# Must either;
|
||||
# - return a Response object
|
||||
# - return a Request object
|
||||
# - or raise IgnoreRequest
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
# Called when a download handler or a process_request()
|
||||
# (from other downloader middleware) raises an exception.
|
||||
|
||||
# Must either:
|
||||
# - return None: continue processing this exception
|
||||
# - return a Response object: stops process_exception() chain
|
||||
# - return a Request object: stops process_exception() chain
|
||||
pass
|
||||
|
||||
def spider_opened(self, spider):
|
||||
spider.logger.info('Spider opened: %s' % spider.name)
|
||||
|
||||
|
||||
class MyUserAgentMiddleware(UserAgentMiddleware):
|
||||
'''
|
||||
设置User-Agent
|
||||
'''
|
||||
|
||||
def __init__(self, user_agent):
|
||||
self.user_agent = user_agent
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(
|
||||
user_agent=crawler.settings.get('USER_AGENTS')
|
||||
)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
agent = random.choice(self.user_agent)
|
||||
request.headers['User-Agent'] = agent
|
||||
print(agent)
|
||||
|
||||
|
||||
class ProxyMiddleware(object):
|
||||
"""docstring for ProxyMiddleWare"""
|
||||
|
||||
def process_request(self, request, spider):
|
||||
'''对request对象加上proxy'''
|
||||
proxy = self.get_random_proxy()
|
||||
print("this is request ip:" + proxy)
|
||||
request.meta['proxy'] = proxy
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
'''对返回的response处理'''
|
||||
# 如果返回的response状态不是200，重新生成当前request对象
|
||||
if response.status != 200:
|
||||
proxy = self.get_random_proxy()
|
||||
print("this is response ip:" + proxy)
|
||||
# 对当前reque加上代理
|
||||
request.meta['proxy'] = proxy
|
||||
return request
|
||||
return response
|
||||
|
||||
def get_random_proxy(self):
|
||||
'''随机从文件中读取proxy'''
|
||||
|
||||
while 1:
|
||||
with open('G:\workspace\common\proxies.txt', 'r') as f:
|
||||
proxies = f.readlines()
|
||||
if proxies:
|
||||
break
|
||||
else:
|
||||
time.sleep(1)
|
||||
proxy = random.choice(proxies).strip()
|
||||
return proxy
|
||||
@@ -0,0 +1,131 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
|
||||
from twisted.enterprise import adbapi
|
||||
import pymysql
|
||||
import pymysql.cursors
|
||||
import time
|
||||
|
||||
|
||||
# class Liepinspd2Pipeline(object):
|
||||
# def __init__(self, dbpool):
|
||||
# self.dbpool = dbpool
|
||||
#
|
||||
# @classmethod
|
||||
# def from_settings(cls, settings): # 函数名固定，会被scrapy调用，直接可用settings的值
|
||||
# """
|
||||
# 数据库建立连接
|
||||
# :param settings: 配置参数
|
||||
# :return: 实例化参数
|
||||
# """
|
||||
#
|
||||
# adbparams = dict(
|
||||
# host=settings['MYSQL_HOST'],
|
||||
# db=settings['MYSQL_DBNAME'],
|
||||
# user=settings['MYSQL_USER'],
|
||||
# password=settings['MYSQL_PASSWORD'],
|
||||
# cursorclass=pymysql.cursors.DictCursor # 指定cursor类型
|
||||
# )
|
||||
# # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
|
||||
# dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
|
||||
# # 返回实例化参数
|
||||
# return cls(dbpool)
|
||||
#
|
||||
# def process_item(self, item, spider):
|
||||
# """
|
||||
# 使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
|
||||
# """
|
||||
# query = self.dbpool.runInteraction(self.do_insert, item) # 指定操作方法和操作数据
|
||||
# # 添加异常处理
|
||||
# query.addCallback(self.handle_error) # 处理异常
|
||||
#
|
||||
# def do_insert(self, cursor, item):
|
||||
# # 对数据库进行插入操作，并不需要commit，twisted会自动commit
|
||||
# insert_sql = "insert into liepin_job(as_of_date,ticker,company_name,job_name,salary,city,education,work_year,pub_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
|
||||
# cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),
|
||||
# str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['origin_site'])))
|
||||
#
|
||||
# def handle_error(self, failure):
|
||||
# if failure:
|
||||
# # 打印错误信息
|
||||
# print(failure)
|
||||
|
||||
class LiepinspecialcomjobPipeline(object):
|
||||
"""
|
||||
同步操作
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# 建立连接
|
||||
self.conn = pymysql.connect('rm-2zewagytttzk6f24xno.mysql.rds.aliyuncs.com', 'cn_ainvest_db', 'cn_ainvest_sd3a1', 'special_data') # 有中文要存入数据库的话要加charset='utf8'
|
||||
# 创建游标
|
||||
self.cursor = self.conn.cursor()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
# sql语句
|
||||
insert_sql = 'insert into job_info(as_of_date,ticker,company_name,job_name,salary,city,education,work_year,pub_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
|
||||
# 执行插入数据到数据库操作
|
||||
self.cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),
|
||||
str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['origin_site'])))
|
||||
# 提交，不进行提交无法保存到数据库
|
||||
self.conn.commit()
|
||||
|
||||
def close_spider(self, spider):
|
||||
# 关闭游标和连接
|
||||
self.cursor.close()
|
||||
self.conn.close()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#
|
||||
#
|
||||
# class LiepinspecialcomjobPipeline(object):
|
||||
# def __init__(self, dbpool):
|
||||
# self.dbpool = dbpool
|
||||
#
|
||||
# @classmethod
|
||||
# def from_settings(cls, settings): # 函数名固定，会被scrapy调用，直接可用settings的值
|
||||
# """
|
||||
# 数据库建立连接
|
||||
# :param settings: 配置参数
|
||||
# :return: 实例化参数
|
||||
# """
|
||||
#
|
||||
# adbparams = dict(
|
||||
# host=settings['MYSQL_HOST'],
|
||||
# db=settings['MYSQL_DBNAME'],
|
||||
# user=settings['MYSQL_USER'],
|
||||
# password=settings['MYSQL_PASSWORD'],
|
||||
# cursorclass=pymysql.cursors.DictCursor # 指定cursor类型
|
||||
# )
|
||||
# # 连接数据池ConnectionPool，使用pymysql或者Mysqldb连接
|
||||
# dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
|
||||
# # 返回实例化参数
|
||||
# return cls(dbpool)
|
||||
#
|
||||
# def process_item(self, item, spider):
|
||||
# """
|
||||
# 使用twisted将MySQL插入变成异步执行。通过连接池执行具体的sql操作，返回一个对象
|
||||
# """
|
||||
# query = self.dbpool.runInteraction(self.do_insert, item) # 指定操作方法和操作数据
|
||||
# # 添加异常处理
|
||||
# query.addCallback(self.handle_error) # 处理异常
|
||||
#
|
||||
# def do_insert(self, cursor, item):
|
||||
# # 对数据库进行插入操作，并不需要commit，twisted会自动commit
|
||||
# insert_sql = "insert into liepin_job(as_of_date,ticker,company_name,job_name,salary,city,education,work_year,pub_time,origin_site) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
|
||||
# cursor.execute(insert_sql, (item['as_of_date'], str(item['ticker']), str(item['company_name']), str(item['job_name']),
|
||||
# str(item['salary']),str(item['city']),str(item['education']),str(item['work_year']),str(item['pub_time']),str(item['origin_site'])))
|
||||
#
|
||||
# def handle_error(self, failure):
|
||||
# if failure:
|
||||
# # 打印错误信息
|
||||
# print(failure)
|
||||
@@ -0,0 +1,179 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Scrapy settings for liepinSpecialComJob project
|
||||
#
|
||||
# For simplicity, this file contains only settings considered important or
|
||||
# commonly used. You can find more settings consulting the documentation:
|
||||
#
|
||||
# https://doc.scrapy.org/en/latest/topics/settings.html
|
||||
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
|
||||
BOT_NAME = 'liepinSpecialComJob'
|
||||
|
||||
SPIDER_MODULES = ['liepinSpecialComJob.spiders']
|
||||
NEWSPIDER_MODULE = 'liepinSpecialComJob.spiders'
|
||||
|
||||
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
#USER_AGENT = 'liepinSpecialComJob (+http://www.yourdomain.com)'
|
||||
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = False
|
||||
|
||||
# Configure maximum concurrent requests performed by Scrapy (default: 16)
|
||||
#CONCURRENT_REQUESTS = 32
|
||||
|
||||
# Configure a delay for requests for the same website (default: 0)
|
||||
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
|
||||
# See also autothrottle settings and docs
|
||||
DOWNLOAD_DELAY = 3
|
||||
# The download delay setting will honor only one of:
|
||||
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
||||
#CONCURRENT_REQUESTS_PER_IP = 16
|
||||
|
||||
# Disable cookies (enabled by default)
|
||||
#COOKIES_ENABLED = False
|
||||
|
||||
# Disable Telnet Console (enabled by default)
|
||||
#TELNETCONSOLE_ENABLED = False
|
||||
|
||||
# Override the default request headers:
|
||||
MYSQL_HOST = 'localhost'
|
||||
MYSQL_DBNAME = 'day0123'
|
||||
MYSQL_USER = 'root'
|
||||
MYSQL_PASSWD = '123'
|
||||
|
||||
DEFAULT_REQUEST_HEADERS = {
|
||||
'Connection': 'keep-alive',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9'
|
||||
}
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5",
|
||||
"MQQBrowser/25 (Linux; U; 2.3.3; zh-cn; HTC Desire S Build/GRI40;480*800)",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.3.3; zh-cn; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1 /110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413",
|
||||
"Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
|
||||
"Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
|
||||
"Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
|
||||
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
|
||||
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
|
||||
"Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
|
||||
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
|
||||
"Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
|
||||
"Mozilla/5.0 (iPod; U; CPU like Mac OS X; en) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/3A101a Safari/419.3",
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
|
||||
'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
|
||||
'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
|
||||
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
|
||||
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
|
||||
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
|
||||
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
|
||||
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
|
||||
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
||||
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
|
||||
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
|
||||
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
|
||||
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
|
||||
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
|
||||
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
|
||||
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
|
||||
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
|
||||
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
|
||||
"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
|
||||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
|
||||
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||
]
|
||||
|
||||
# Enable or disable spider middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
#SPIDER_MIDDLEWARES = {
|
||||
# 'liepinSpecialComJob.middlewares.LiepinspecialcomjobSpiderMiddleware': 543,
|
||||
#}
|
||||
|
||||
# Enable or disable downloader middlewares
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
# 'liepinSpd2.middlewares.Liepinspd2DownloaderMiddleware': 543,
|
||||
'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
|
||||
'liepinSpd2.middlewares.MyUserAgentMiddleware': 400,
|
||||
# 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': None,
|
||||
# 'liepinSpd2.middlewares.ProxyMiddleware': 750,
|
||||
# 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': None,
|
||||
}
|
||||
|
||||
ITEM_PIPELINES = {
|
||||
'liepinSpecialComJob.pipelines.LiepinspecialcomjobPipeline': 300,
|
||||
}
|
||||
|
||||
# Enable or disable extensions
|
||||
# See https://doc.scrapy.org/en/latest/topics/extensions.html
|
||||
#EXTENSIONS = {
|
||||
# 'scrapy.extensions.telnet.TelnetConsole': None,
|
||||
#}
|
||||
|
||||
# Configure item pipelines
|
||||
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
|
||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
|
||||
#AUTOTHROTTLE_ENABLED = True
|
||||
# The initial download delay
|
||||
#AUTOTHROTTLE_START_DELAY = 5
|
||||
# The maximum download delay to be set in case of high latencies
|
||||
#AUTOTHROTTLE_MAX_DELAY = 60
|
||||
# The average number of requests Scrapy should be sending in parallel to
|
||||
# each remote server
|
||||
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||
# Enable showing throttling stats for every response received:
|
||||
#AUTOTHROTTLE_DEBUG = False
|
||||
|
||||
# Enable and configure HTTP caching (disabled by default)
|
||||
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||
#HTTPCACHE_ENABLED = True
|
||||
#HTTPCACHE_EXPIRATION_SECS = 0
|
||||
#HTTPCACHE_DIR = 'httpcache'
|
||||
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
@@ -0,0 +1,4 @@
|
||||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# Please refer to the documentation for information on how to create and manage
|
||||
# your spiders.
|
||||
@@ -0,0 +1,152 @@
|
||||
import json
|
||||
|
||||
import scrapy
|
||||
import re
|
||||
from datetime import datetime
|
||||
import pandas as pd
|
||||
import time
|
||||
from common.util import get_13_time
|
||||
t = get_13_time()
|
||||
from liepinSpecialComJob.items import LiepinspecialcomjobItem
|
||||
|
||||
|
||||
class LiepinSpdier(scrapy.Spider):
|
||||
name = 'liepin'
|
||||
start_urls = ['https://vip.liepin.com/883905/1405577359643.shtml',
|
||||
'https://vip.liepin.com/8161070/joblist.shtml',
|
||||
# 'http://maker.haier.net/custompage/socialchannel/index.html?platformcode=lp',
|
||||
'https://vip.liepin.com/7855333/joblist.shtml',
|
||||
'https://vip.liepin.com/8090130/1409730340536.shtml',
|
||||
'https://vip.liepin.com/8399212/joblist.shtml',
|
||||
'https://vip.liepin.com/1198424/joblist2.shtml',
|
||||
'https://vip.liepin.com/8787971/joblist.shtml',
|
||||
'https://vip.liepin.com/8796178/joblist2.shtml',
|
||||
'https://vip.liepin.com/8091337/1426475303042.shtml',
|
||||
'https://vip.liepin.com/7904788/job.shtml',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
text = response.text
|
||||
company_name = re.search(r'<title>(.*?) - 猎聘网招聘官网',text).group(1)
|
||||
companyId=re.search(r'CONFIG={"companyId":"([0-9]+)"}',text).group(1)
|
||||
next_meta = response.meta
|
||||
data = pd.read_csv('G:\workspace\y2019m01\/first_lagou\company300.csv', encoding='gbk')
|
||||
try:
|
||||
for i in range(len(data)):
|
||||
n = 0
|
||||
for j in data.loc[i, '股票简称']:
|
||||
if j in company_name:
|
||||
n += 1
|
||||
if n == len(data.loc[i, '股票简称']):
|
||||
next_meta['ticker'] = data.loc[i, '股票代码']
|
||||
print(n, next_meta['ticker'], company_name)
|
||||
except BaseException as e:
|
||||
next_meta['ticker'] ='None'
|
||||
print('ticker匹配错误')
|
||||
next_meta['company_name'] = company_name
|
||||
next_meta['companyId'] = companyId
|
||||
url='https://www.liepin.com/ajaxproxy.html'
|
||||
# headers={
|
||||
# 'Referer':'https://vip.liepin.com/8091337/1426475303042.shtml'
|
||||
# }
|
||||
yield scrapy.Request(url, callback=self.parse_list, meta=next_meta,dont_filter=True)
|
||||
|
||||
def parse_list(self,response):
|
||||
next_meta = response.meta
|
||||
companyId = next_meta['companyId'].strip()
|
||||
# print(companyId,response.text)
|
||||
n=0
|
||||
while n<95:
|
||||
# try:
|
||||
t = get_13_time()
|
||||
# 'https://www.liepin.com/company/sojob.json?pageSize=15&curPage=0&ecompIds=8091337&dq=&publishTime=&keywords=&_=1550383073951'
|
||||
url=f'https://www.liepin.com/company/sojob.json?pageSize=15&curPage={n}&ecompIds={companyId}&dq=&publishTime=&keywords=&_={t}'
|
||||
n+=1
|
||||
headers={
|
||||
'referer':'https://www.liepin.com/ajaxproxy.html'
|
||||
}
|
||||
cookies={
|
||||
'__uuid': '1550017147980.22',
|
||||
'_uuid': 'E4361B46FFA8441973EC46E6488BD983',
|
||||
'is_lp_user': 'true',
|
||||
'need_bind_tel': 'false',
|
||||
'new_user': 'false',
|
||||
'c_flag': 'f57e19ed294147b87179e4e6132477f5',
|
||||
'imClientId': '45e417dd37f82ac674cdcbb355984626',
|
||||
'imId': '45e417dd37f82ac6a36687782a0c1c67',
|
||||
'imClientId_0': '45e417dd37f82ac674cdcbb355984626',
|
||||
'imId_0': '45e417dd37f82ac6a36687782a0c1c67',
|
||||
'gr_user_id': '374534ce-aa54-4880-88ca-7a7bb7adf340',
|
||||
'bad1b2d9162fab1f80dde1897f7a2972_gr_last_sent_cs1': '463d81f04fd219c61a667e00ad0d9493',
|
||||
'grwng_uid': 'f3fda8f8-0c2e-4f29-8507-f42f7a9671ec',
|
||||
'fe_work_exp_add': 'true',
|
||||
'ADHOC_MEMBERSHIP_CLIENT_ID1.0': 'fa804ff0-2a02-3f31-8dcb-8e13b527dfcb',
|
||||
'bad1b2d9162fab1f80dde1897f7a2972_gr_cs1': '463d81f04fd219c61a667e00ad0d9493',
|
||||
'__tlog': '1550383052778.97%7C00000000%7C00000000%7C00000000%7C00000000',
|
||||
'_mscid': '00000000',
|
||||
'Hm_lvt_a2647413544f5a04f00da7eee0d5e200': '1550233873,1550279247,1550281552,1550383053',
|
||||
'abtest': '0',
|
||||
'_fecdn_': '0',
|
||||
'__session_seq': '2',
|
||||
'__uv_seq': '2',
|
||||
'Hm_lpvt_a2647413544f5a04f00da7eee0d5e200': '1550383074'
|
||||
}
|
||||
next_meta['ticker'] = next_meta['ticker']
|
||||
print(next_meta['ticker'])
|
||||
next_meta['company_name'] = next_meta['company_name']
|
||||
print(next_meta['company_name'])
|
||||
yield scrapy.Request(url, callback=self.parse_job,meta=next_meta,headers=headers,cookies=cookies)
|
||||
# except BaseException as e:
|
||||
# print('已完成最后一页')
|
||||
# break
|
||||
|
||||
def parse_job(self,response):
|
||||
meta = response.meta
|
||||
item = LiepinspecialcomjobItem()
|
||||
text = response.text
|
||||
print('****************************************')
|
||||
json_data = json.loads(text)
|
||||
as_of_date = datetime.now()
|
||||
job_infos=json_data['list']
|
||||
for job_info in job_infos:
|
||||
origin_site=job_info['url']
|
||||
job_name=job_info['title']
|
||||
salary=job_info['salary']
|
||||
city=job_info['city']
|
||||
education=job_info['eduLevel']
|
||||
work_year=job_info['workYear']
|
||||
pub_time=job_info['time']
|
||||
function=job_info['dept']
|
||||
|
||||
item['ticker'] = meta['ticker'].strip()
|
||||
item['company_name'] = meta['company_name'].strip()
|
||||
item['job_name']=job_name
|
||||
item['salary']=salary
|
||||
item['city']=city
|
||||
item['education']=education
|
||||
item['work_year']=work_year
|
||||
item['pub_time']=pub_time
|
||||
item['as_of_date']=as_of_date
|
||||
item['function']=function
|
||||
item['origin_site']=origin_site
|
||||
|
||||
yield item
|
||||
|
||||
|
||||
|
||||
#暂不深挖
|
||||
# for url in origin_sites:
|
||||
# yield scrapy.Request(url, callback=self.parse_job)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# 获取settings.py模块的设置
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
from liepinSpecialComJob.spiders.lpspecialcomjob import LiepinSpdier
|
||||
|
||||
settings = get_project_settings()
|
||||
process = CrawlerProcess(settings=settings)
|
||||
|
||||
# 可以添加多个spider类
|
||||
process.crawl(LiepinSpdier)
|
||||
|
||||
# 启动爬虫，会阻塞，直到爬取完成
|
||||
process.start()
|
||||
@@ -0,0 +1,11 @@
|
||||
# Automatically created by: scrapy startproject
|
||||
#
|
||||
# For more information about the [deploy] section see:
|
||||
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
|
||||
[settings]
|
||||
default = liepinSpecialComJob.settings
|
||||
|
||||
[deploy]
|
||||
#url = http://localhost:6800/
|
||||
project = liepinSpecialComJob
|
||||
Reference in New Issue
Block a user