添加链家spider
This commit is contained in:
criseLYJ
Vendored
BIN
Binary file not shown.
@@ -44,6 +44,11 @@
|
||||
|
||||
## 模拟登录一些常见的网站和爬取相应的信息
|
||||
|
||||
## Have something to say
|
||||
- 今天有人给我说了一些东西，他说：”你今天开源这些东西会让别人容易的反爬虫“。对，我知道，当然我很清楚这些，但是人生总是需要一些挑战不是吗？兄弟？我认为当我攻破他们防守的时刻，才是最刺激的，不是吗？如果没人反抗，我做这些才是真的毫无意义，只有真正的攻与守，才能促使我们进步，时代在变兄弟，科技也在变，安于现状，才是错误的根本原因！我今年才21岁，我还有大把的时间，我会持续维护此网站，接受所有的挑战！当然我也需要一份稳定的工作！
|
||||
- 希望你们不要安于现状，生活处处是惊喜！
|
||||
|
||||
### Life is fantastic! bro~
|
||||
|
||||
## About
|
||||
|
||||
@@ -99,6 +104,10 @@
|
||||
|
||||
|
||||
|
||||
### 新增链家Spider
|
||||
|
||||
|
||||
|
||||
```
|
||||
1. 爬取淘宝各子标签，按销量排名商品信息，按分类保存至MongoDB
|
||||
2. 通过pandas进行数据分析
|
||||
@@ -149,10 +158,11 @@
|
||||
- 如果关注量大的话，我还是会不断维护此仓库带来更多的东西，并且重构代码，
|
||||
|
||||
## Acknowledgments
|
||||
- Thanks for all!
|
||||
### Thanks for all!
|
||||
|
||||
|
||||
|
||||
### 在这里还是要特别感谢V2EX上的所有人！多谢🙏,也感谢从其他地方过来的所有人，感谢🙏
|
||||
### 再次感谢所有支持我的人---
|
||||
|
||||
## Written at the end
|
||||
- I need your support.
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 324 KiB |
@@ -0,0 +1,6 @@
|
||||
一个基于分页、线程池、代理池的链家网快速爬虫项目，速度可达 10000 条/5 分钟，严禁将所得数据商用！
|
||||
|
||||
同时对数据进行了清洗、分析、可视化。
|
||||
|
||||
- lianjia:只爬取
|
||||
- pic：图表
|
||||
@@ -0,0 +1,191 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import requests
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from pyquery import PyQuery as pq
|
||||
|
||||
import json
|
||||
|
||||
import threading
|
||||
|
||||
import time
|
||||
|
||||
"""
|
||||
info:
|
||||
author:CriseLYJ
|
||||
github:https://github.com/CriseLYJ/
|
||||
update_time:2019-3-7
|
||||
"""
|
||||
|
||||
|
||||
def get_list_page_url(city):
|
||||
start_url = "https://{}.lianjia.com/ershoufang".format(city)
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
|
||||
}
|
||||
try:
|
||||
response = requests.get(start_url, headers=headers)
|
||||
# print(response.status_code, response.text)
|
||||
doc = pq(response.text)
|
||||
total_num = int(doc(".resultDes .total span").text())
|
||||
total_page = total_num // 30 + 1
|
||||
# 只能访问到前一百页
|
||||
if total_page > 100:
|
||||
total_page = 100
|
||||
|
||||
page_url_list = list()
|
||||
|
||||
for i in range(total_page):
|
||||
url = start_url + "/pg" + str(i + 1) + "/"
|
||||
page_url_list.append(url)
|
||||
# print(url)
|
||||
return page_url_list
|
||||
|
||||
except:
|
||||
print("获取总套数出错,请确认起始URL是否正确")
|
||||
return None
|
||||
|
||||
|
||||
detail_list = list()
|
||||
|
||||
|
||||
# 需要先在本地开启代理池
|
||||
# 代理池仓库: https://github.com/Python3WebSpider/ProxyPool
|
||||
def get_valid_ip():
|
||||
url = "http://localhost:5000/get"
|
||||
try:
|
||||
ip = requests.get(url).text
|
||||
return ip
|
||||
except:
|
||||
print("请先运行代理池")
|
||||
|
||||
|
||||
def get_detail_page_url(page_url):
|
||||
global detail_list
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
|
||||
'Referer': 'https://bj.lianjia.com/ershoufang'
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(page_url, headers=headers, timeout=3)
|
||||
doc = pq(response.text)
|
||||
# broswer.get(page_url)
|
||||
# print(page_url)
|
||||
# doc = pq(broswer.page_source)
|
||||
i = 0
|
||||
detail_urls = list()
|
||||
for item in doc(".sellListContent li").items():
|
||||
i += 1
|
||||
if i == 31:
|
||||
break
|
||||
child_item = item(".noresultRecommend")
|
||||
if child_item == None:
|
||||
i -= 1
|
||||
detail_url = child_item.attr("href")
|
||||
detail_urls.append(detail_url)
|
||||
return detail_urls
|
||||
except:
|
||||
print("获取列表页" + page_url + "出错")
|
||||
|
||||
|
||||
lock = threading.Lock()
|
||||
|
||||
|
||||
def detail_page_parser(res):
|
||||
global detail_list
|
||||
detail_urls = res.result()
|
||||
if not detail_urls:
|
||||
print("detail url 为空")
|
||||
return None
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
|
||||
'Referer': 'https://bj.lianjia.com/ershoufang'
|
||||
}
|
||||
for detail_url in detail_urls:
|
||||
try:
|
||||
response = requests.get(url=detail_url, headers=headers, timeout=3)
|
||||
# print(response.status_code)
|
||||
detail_dict = dict()
|
||||
doc = pq(response.text)
|
||||
unit_price = doc(".unitPriceValue").text()
|
||||
unit_price = unit_price[0:unit_price.index("元")]
|
||||
title = doc("h1").text()
|
||||
area = doc(".areaName .info a").eq(0).text().strip()
|
||||
url = detail_url
|
||||
detail_dict["title"] = title
|
||||
detail_dict["area"] = area
|
||||
detail_dict["price"] = unit_price
|
||||
detail_dict["url"] = url
|
||||
|
||||
detail_list.append(detail_dict)
|
||||
|
||||
print(unit_price, title, area)
|
||||
|
||||
except:
|
||||
print("获取详情页出错,换ip重试")
|
||||
proxies = {
|
||||
"http": "http://" + get_valid_ip(),
|
||||
}
|
||||
try:
|
||||
response = requests.get(url=detail_url, headers=headers, proxies=proxies)
|
||||
# print(response.status_code)
|
||||
detail_dict = dict()
|
||||
doc = pq(response.text)
|
||||
unit_price = doc(".unitPriceValue").text()
|
||||
unit_price = unit_price[0:unit_price.index("元")]
|
||||
title = doc("h1").text()
|
||||
area = doc(".areaName .info a").eq(0).text().strip()
|
||||
url = detail_url
|
||||
# 已下架的还会爬取，但是没有价格
|
||||
if len(unit_price) > 0:
|
||||
detail_dict["title"] = title
|
||||
detail_dict["area"] = area
|
||||
detail_dict["price"] = unit_price
|
||||
detail_dict["url"] = url
|
||||
|
||||
detail_list.append(detail_dict)
|
||||
|
||||
print(unit_price, title, area)
|
||||
except:
|
||||
print("重试失败...")
|
||||
|
||||
|
||||
def save_data(data, filename):
|
||||
with open(filename + ".json", 'w', encoding="utf-8") as f:
|
||||
f.write(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
def main():
|
||||
# cq,cs,nj,dl,wh,cc
|
||||
city_list = ['nj']
|
||||
for city in city_list:
|
||||
page_url_list = get_list_page_url(city)
|
||||
|
||||
# pool = threadpool.ThreadPool(20)
|
||||
# requests = threadpool.makeRequests(page_and_detail_parser, page_url_list)
|
||||
# [pool.putRequest(req) for req in requests]
|
||||
# pool.wait()
|
||||
|
||||
p = ThreadPoolExecutor(30)
|
||||
|
||||
for page_url in page_url_list:
|
||||
p.submit(get_detail_page_url, page_url).add_done_callback(detail_page_parser)
|
||||
# 这里的回调函数拿到的是一个对象。
|
||||
# 先把返回的res得到一个结果。即在前面加上一个res.result(),这个结果就是get_detail_page_url的返回
|
||||
p.shutdown()
|
||||
|
||||
print(detail_list)
|
||||
|
||||
save_data(detail_list, city)
|
||||
|
||||
detail_list.clear()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
old = time.time()
|
||||
main()
|
||||
new = time.time()
|
||||
delta_time = new - old
|
||||
print("程序共运行{}s".format(delta_time))
|
||||
@@ -0,0 +1,103 @@
|
||||
import json
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib
|
||||
|
||||
# 设置中文字体和负号正常显示
|
||||
matplotlib.rcParams['font.sans-serif'] = ['SimHei']
|
||||
matplotlib.rcParams['axes.unicode_minus'] = False
|
||||
|
||||
"""
|
||||
info:
|
||||
author:CriseLYJ
|
||||
github:https://github.com/CriseLYJ/
|
||||
update_time:2019-3-7
|
||||
"""
|
||||
|
||||
|
||||
def load_data():
|
||||
global data, dic_data
|
||||
with open("xm.json", "r", encoding="utf-8") as f:
|
||||
str = f.read()
|
||||
data = json.loads(str)
|
||||
dic_data = dict()
|
||||
for i in range(len(data)):
|
||||
per_price = (int)(data[i].get("price"))
|
||||
region = data[i].get("area")
|
||||
# print(per_price,region)
|
||||
try:
|
||||
dic_data[region].append(per_price)
|
||||
except:
|
||||
dic_data[region] = [per_price]
|
||||
print(dic_data)
|
||||
|
||||
|
||||
def split_data():
|
||||
global region_data
|
||||
region_data = dict()
|
||||
for region in dic_data.keys():
|
||||
# 最大值、最小值、平均值
|
||||
region_data[region] = {"max": dic_data[region][0], "min": dic_data[region][0], "average": 0}
|
||||
for per_price in dic_data[region]:
|
||||
if per_price > region_data[region]["max"]:
|
||||
region_data[region]["max"] = per_price
|
||||
if per_price < region_data[region]["min"]:
|
||||
region_data[region]["min"] = per_price
|
||||
region_data[region]["average"] += per_price
|
||||
region_data[region]["average"] /= len(dic_data[region])
|
||||
# 保留两位小数
|
||||
region_data[region]["average"] = round(region_data[region]["average"], 2)
|
||||
print(region_data)
|
||||
|
||||
|
||||
def data_viewer():
|
||||
label_list = region_data.keys() # 横坐标刻度显示值
|
||||
max = []
|
||||
min = []
|
||||
average = []
|
||||
for label in label_list:
|
||||
max.append(region_data[label].get("max"))
|
||||
min.append(region_data[label].get("min"))
|
||||
average.append(region_data[label].get("average"))
|
||||
x = range(len(max))
|
||||
"""
|
||||
绘制条形图
|
||||
left: 长条形中点横坐标
|
||||
height: 长条形高度
|
||||
width: 长条形宽度，默认值0
|
||||
.8
|
||||
label: 为后面设置legend准备
|
||||
"""
|
||||
rects1 = plt.bar(x=x, height=max, width=0.25, alpha=0.8, color='red', label="最大值")
|
||||
rects2 = plt.bar(x=[i + 0.25 for i in x], height=average, width=0.25, color='green', label="平均值")
|
||||
rects3 = plt.bar(x=[i + 0.5 for i in x], height=min, width=0.25, color='blue', label="最小值")
|
||||
# plt.ylim(0, 50) # y轴取值范围
|
||||
plt.ylabel("房价/元")
|
||||
"""
|
||||
设置x轴刻度显示值
|
||||
参数一：中点坐标
|
||||
参数二：显示值
|
||||
"""
|
||||
plt.xticks([index + 0.25 for index in x], label_list)
|
||||
plt.xlabel("区")
|
||||
plt.title("厦门二手房价格分析图")
|
||||
plt.legend() # 设置题注 # 编辑文本
|
||||
for rect in rects1:
|
||||
height = rect.get_height()
|
||||
plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom")
|
||||
for rect in rects2:
|
||||
height = rect.get_height()
|
||||
plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom")
|
||||
for rect in rects3:
|
||||
height = rect.get_height()
|
||||
plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom")
|
||||
plt.show()
|
||||
|
||||
|
||||
def main():
|
||||
load_data()
|
||||
split_data()
|
||||
data_viewer()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user