diff --git a/.DS_Store b/.DS_Store index f01e028..ffaa6ab 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/Github/.DS_Store b/Github/.DS_Store index 55cc84b..fce0419 100644 Binary files a/Github/.DS_Store and b/Github/.DS_Store differ diff --git a/README.md b/README.md index d7b152c..c85da42 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,11 @@ ## 模拟登录一些常见的网站和爬取相应的信息 +## Have something to say +- 今天有人给我说了一些东西，他说：”你今天开源这些东西会让别人容易的反爬虫“。对，我知道，当然我很清楚这些，但是人生总是需要一些挑战不是吗？兄弟？我认为当我攻破他们防守的时刻，才是最刺激的，不是吗？如果没人反抗，我做这些才是真的毫无意义，只有真正的攻与守，才能促使我们进步，时代在变兄弟，科技也在变，安于现状，才是错误的根本原因！我今年才21岁，我还有大把的时间，我会持续维护此网站，接受所有的挑战！当然我也需要一份稳定的工作！ +- 希望你们不要安于现状，生活处处是惊喜！ + +### Life is fantastic! bro~ ## About @@ -99,6 +104,10 @@ ![](./image/github.jpg) +### 新增链家Spider + +![](./image/lianjia.jpg) + ``` 1. 爬取淘宝各子标签，按销量排名商品信息，按分类保存至MongoDB 2. 通过pandas进行数据分析 @@ -149,10 +158,11 @@ - 如果关注量大的话，我还是会不断维护此仓库带来更多的东西，并且重构代码， ## Acknowledgments -- Thanks for all! +### Thanks for all! + ![](./image/yanjing.png) -### 在这里还是要特别感谢V2EX上的所有人！多谢🙏,也感谢从其他地方过来的所有人，感谢🙏 +### 再次感谢所有支持我的人--- ## Written at the end - I need your support. diff --git a/image/lianjia.jpg b/image/lianjia.jpg new file mode 100644 index 0000000..21f41bc Binary files /dev/null and b/image/lianjia.jpg differ diff --git a/lianjia_spider/README.md b/lianjia_spider/README.md new file mode 100644 index 0000000..7f66a9f --- /dev/null +++ b/lianjia_spider/README.md @@ -0,0 +1,6 @@ +一个基于分页、线程池、代理池的链家网快速爬虫项目，速度可达 10000 条/5 分钟，严禁将所得数据商用！ + +同时对数据进行了清洗、分析、可视化。 + +- lianjia:只爬取 +- pic：图表 diff --git a/lianjia_spider/lianjia.py b/lianjia_spider/lianjia.py new file mode 100644 index 0000000..7b20b4d --- /dev/null +++ b/lianjia_spider/lianjia.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +import requests + +from concurrent.futures import ThreadPoolExecutor + +from pyquery import PyQuery as pq + +import json + +import threading + +import time + +""" +info: +author:CriseLYJ +github:https://github.com/CriseLYJ/ +update_time:2019-3-7 +""" + + +def get_list_page_url(city): + start_url = "https://{}.lianjia.com/ershoufang".format(city) + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36', + } + try: + response = requests.get(start_url, headers=headers) + # print(response.status_code, response.text) + doc = pq(response.text) + total_num = int(doc(".resultDes .total span").text()) + total_page = total_num // 30 + 1 + # 只能访问到前一百页 + if total_page > 100: + total_page = 100 + + page_url_list = list() + + for i in range(total_page): + url = start_url + "/pg" + str(i + 1) + "/" + page_url_list.append(url) + # print(url) + return page_url_list + + except: + print("获取总套数出错,请确认起始URL是否正确") + return None + + +detail_list = list() + + +# 需要先在本地开启代理池 +# 代理池仓库: https://github.com/Python3WebSpider/ProxyPool +def get_valid_ip(): + url = "http://localhost:5000/get" + try: + ip = requests.get(url).text + return ip + except: + print("请先运行代理池") + + +def get_detail_page_url(page_url): + global detail_list + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36', + 'Referer': 'https://bj.lianjia.com/ershoufang' + } + + try: + response = requests.get(page_url, headers=headers, timeout=3) + doc = pq(response.text) + # broswer.get(page_url) + # print(page_url) + # doc = pq(broswer.page_source) + i = 0 + detail_urls = list() + for item in doc(".sellListContent li").items(): + i += 1 + if i == 31: + break + child_item = item(".noresultRecommend") + if child_item == None: + i -= 1 + detail_url = child_item.attr("href") + detail_urls.append(detail_url) + return detail_urls + except: + print("获取列表页" + page_url + "出错") + + +lock = threading.Lock() + + +def detail_page_parser(res): + global detail_list + detail_urls = res.result() + if not detail_urls: + print("detail url 为空") + return None + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36', + 'Referer': 'https://bj.lianjia.com/ershoufang' + } + for detail_url in detail_urls: + try: + response = requests.get(url=detail_url, headers=headers, timeout=3) + # print(response.status_code) + detail_dict = dict() + doc = pq(response.text) + unit_price = doc(".unitPriceValue").text() + unit_price = unit_price[0:unit_price.index("元")] + title = doc("h1").text() + area = doc(".areaName .info a").eq(0).text().strip() + url = detail_url + detail_dict["title"] = title + detail_dict["area"] = area + detail_dict["price"] = unit_price + detail_dict["url"] = url + + detail_list.append(detail_dict) + + print(unit_price, title, area) + + except: + print("获取详情页出错,换ip重试") + proxies = { + "http": "http://" + get_valid_ip(), + } + try: + response = requests.get(url=detail_url, headers=headers, proxies=proxies) + # print(response.status_code) + detail_dict = dict() + doc = pq(response.text) + unit_price = doc(".unitPriceValue").text() + unit_price = unit_price[0:unit_price.index("元")] + title = doc("h1").text() + area = doc(".areaName .info a").eq(0).text().strip() + url = detail_url + # 已下架的还会爬取，但是没有价格 + if len(unit_price) > 0: + detail_dict["title"] = title + detail_dict["area"] = area + detail_dict["price"] = unit_price + detail_dict["url"] = url + + detail_list.append(detail_dict) + + print(unit_price, title, area) + except: + print("重试失败...") + + +def save_data(data, filename): + with open(filename + ".json", 'w', encoding="utf-8") as f: + f.write(json.dumps(data, indent=2, ensure_ascii=False)) + + +def main(): + # cq,cs,nj,dl,wh,cc + city_list = ['nj'] + for city in city_list: + page_url_list = get_list_page_url(city) + + # pool = threadpool.ThreadPool(20) + # requests = threadpool.makeRequests(page_and_detail_parser, page_url_list) + # [pool.putRequest(req) for req in requests] + # pool.wait() + + p = ThreadPoolExecutor(30) + + for page_url in page_url_list: + p.submit(get_detail_page_url, page_url).add_done_callback(detail_page_parser) + # 这里的回调函数拿到的是一个对象。 + # 先把返回的res得到一个结果。即在前面加上一个res.result(),这个结果就是get_detail_page_url的返回 + p.shutdown() + + print(detail_list) + + save_data(detail_list, city) + + detail_list.clear() + + +if __name__ == '__main__': + old = time.time() + main() + new = time.time() + delta_time = new - old + print("程序共运行{}s".format(delta_time)) diff --git a/lianjia_spider/pic.py b/lianjia_spider/pic.py new file mode 100644 index 0000000..0a922f5 --- /dev/null +++ b/lianjia_spider/pic.py @@ -0,0 +1,103 @@ +import json +import matplotlib.pyplot as plt +import matplotlib + +# 设置中文字体和负号正常显示 +matplotlib.rcParams['font.sans-serif'] = ['SimHei'] +matplotlib.rcParams['axes.unicode_minus'] = False + +""" +info: +author:CriseLYJ +github:https://github.com/CriseLYJ/ +update_time:2019-3-7 +""" + + +def load_data(): + global data, dic_data + with open("xm.json", "r", encoding="utf-8") as f: + str = f.read() + data = json.loads(str) + dic_data = dict() + for i in range(len(data)): + per_price = (int)(data[i].get("price")) + region = data[i].get("area") + # print(per_price,region) + try: + dic_data[region].append(per_price) + except: + dic_data[region] = [per_price] + print(dic_data) + + +def split_data(): + global region_data + region_data = dict() + for region in dic_data.keys(): + # 最大值、最小值、平均值 + region_data[region] = {"max": dic_data[region][0], "min": dic_data[region][0], "average": 0} + for per_price in dic_data[region]: + if per_price > region_data[region]["max"]: + region_data[region]["max"] = per_price + if per_price < region_data[region]["min"]: + region_data[region]["min"] = per_price + region_data[region]["average"] += per_price + region_data[region]["average"] /= len(dic_data[region]) + # 保留两位小数 + region_data[region]["average"] = round(region_data[region]["average"], 2) + print(region_data) + + +def data_viewer(): + label_list = region_data.keys() # 横坐标刻度显示值 + max = [] + min = [] + average = [] + for label in label_list: + max.append(region_data[label].get("max")) + min.append(region_data[label].get("min")) + average.append(region_data[label].get("average")) + x = range(len(max)) + """ + 绘制条形图 + left: 长条形中点横坐标 + height: 长条形高度 + width: 长条形宽度，默认值0 + .8 + label: 为后面设置legend准备 + """ + rects1 = plt.bar(x=x, height=max, width=0.25, alpha=0.8, color='red', label="最大值") + rects2 = plt.bar(x=[i + 0.25 for i in x], height=average, width=0.25, color='green', label="平均值") + rects3 = plt.bar(x=[i + 0.5 for i in x], height=min, width=0.25, color='blue', label="最小值") + # plt.ylim(0, 50) # y轴取值范围 + plt.ylabel("房价/元") + """ + 设置x轴刻度显示值 + 参数一：中点坐标 + 参数二：显示值 + """ + plt.xticks([index + 0.25 for index in x], label_list) + plt.xlabel("区") + plt.title("厦门二手房价格分析图") + plt.legend() # 设置题注 # 编辑文本 + for rect in rects1: + height = rect.get_height() + plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom") + for rect in rects2: + height = rect.get_height() + plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom") + for rect in rects3: + height = rect.get_height() + plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom") + plt.show() + + +def main(): + load_data() + split_data() + data_viewer() + + +if __name__ == '__main__': + main()