lambert/awesome-python-login-model
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PEP8

Browse Source
This commit is contained in:
criseLYJ
2019-03-08 23:26:36 +08:00
parent eb4fdc2b26
commit 837256702f
23 changed files with 268 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+5
View File
@@ -103,4 +103,9 @@ venv.bak/
# mypy
.mypy_cache/
# Mac os
.DS_Store
# pycharm
.idea
.env
163email/163email.py
+1 -1
View File
@@ -29,4 +29,4 @@ def login():
if __name__ == '__main__':
login()
login()
Github/github_login.py
+6 -5
View File
@@ -15,11 +15,11 @@ class Login(object):
def __init__(self):
base_url = 'https://github.com/'
# 登陆 url
self.login_url = base_url +'login'
self.login_url = base_url + 'login'
# 提交表单的 api
self.post_url = base_url +'session'
self.post_url = base_url + 'session'
# 个人资料页面的 url
self.logined_url = base_url +'settings/profile'
self.logined_url = base_url + 'settings/profile'
# 构造一个会话对象
self.session = requests.Session()
# 自定义请求头
@@ -36,7 +36,7 @@ class Login(object):
doc = pq(response.text)
token = doc('input[name="authenticity_token"]').attr("value").strip()
return token
def login(self, email, password):
token = self.token()
# 构造表单数据
@@ -71,7 +71,7 @@ class Login(object):
Repositories = doc("div.Box-body > ul > li").text().split()
for Repositorie in Repositories:
print(Repositorie)
def profile(self, html):
doc = pq(html)
page_title = doc("title").text()
@@ -89,6 +89,7 @@ class Login(object):
password = getpass.getpass("password:")
self.login(email=email, password=password)
if __name__ == "__main__":
login = Login()
login.main()
Github/github_login2.py
+70
View File
@@ -0,0 +1,70 @@
# -*- coding: utf-8 -*-
"""
github第二种登录方式
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-7
"""
import requests
from lxml import etree
class Login(object):
def __init__(self, email, password):
self.headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
'Referer': 'https://github.com/',
'Host': 'github.com'
}
self.login_url = 'https://github.com/login'
self.post_url = 'https://github.com/session'
self.session = requests.Session()
self.email = email
self.password = password
# 模拟登录
def login_GitHub(self):
post_data = {
'commit': 'Sign in',
'utf8': '✓',
'authenticity_token': self.get_token(),
'login': self.email,
'password': self.password
}
response = self.session.post(self.post_url, data=post_data, headers=self.headers)
print(response.status_code)
print(post_data)
if response.status_code == 200:
print("登录成功！")
else:
print("登录失败！")
# 获取token信息
def get_token(self):
response = self.session.get(self.login_url, headers=self.headers)
html = etree.HTML(response.content.decode())
token = html.xpath('//input[@name="authenticity_token"]/@value')[0]
return token
if __name__ == '__main__':
email = input('请输入您的账号： ')
password = input('请输入您的密码： ')
login = Login(email, password)
login.login_GitHub()
NeteaseCloudMusicDownload/wangyiyun_spider.py
+28 -30
View File
@@ -3,17 +3,17 @@
# @Date: 2018-12-8 17:13:11
import requests
import math,random
import requests
import math, random
from Crypto.Cipher import AES
import base64
import codecs
import os
import base64
import codecs
import os
class decrypt_music(object):
def __init__(self,d):
self.d = d
def __init__(self, d):
self.d = d
self.e = '010001'
self.f = "00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5a" \
"a76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46be" \
@@ -21,22 +21,21 @@ class decrypt_music(object):
self.g = '0CoJUm6Qyw8W8jud'
self.random_text = self.get_random_str()
def get_random_str(self):
str = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
res = ''
for x in range(16):
for x in range(16):
index = math.floor(random.random() * len(str))
res += str[index]
return res
return res
def aes_encrypt(self,text,key):
def aes_encrypt(self, text, key):
iv = '0102030405060708'
pad = 16 - len(text.encode()) % 16
pad = 16 - len(text.encode()) % 16
text = text + pad * chr(pad)
encryptor = AES.new(key, AES.MODE_CBC, iv)
msg = base64.b64encode(encryptor.encrypt(text))
return msg
return msg
def rsa_encrypt(self, value, text, modulus):
'''进行rsa加密'''
@@ -44,34 +43,33 @@ class decrypt_music(object):
rs = int(codecs.encode(text.encode('utf-8'), 'hex_codec'), 16) ** int(value, 16) % int(modulus, 16)
return format(rs, 'x').zfill(256)
def get_data(self):
params = self.aes_encrypt(self.d,self.g)
params = self.aes_encrypt(params.decode('utf-8'),self.random_text)
enc_sec_key = self.rsa_encrypt(self.e,self.random_text,self.f)
params = self.aes_encrypt(self.d, self.g)
params = self.aes_encrypt(params.decode('utf-8'), self.random_text)
enc_sec_key = self.rsa_encrypt(self.e, self.random_text, self.f)
return {
'params':params,
'encSecKey':enc_sec_key
'params': params,
'encSecKey': enc_sec_key
}
class Spider(object):
def __init__(self):
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0',
'Cookie':'_iuqxldmzr_=32; _ntes_nnid=8d4ef0883a3bcc9d3a2889b0bf36766a,1533782432391; _ntes_nuid=8d4ef0883a3bcc9d3a2889b0bf36766a; __utmc=94650624; WM_TID=GzmBlbRkRGQXeQiYuDVCfoEatU6VSsKC; playerid=19729878; __utma=94650624.1180067615.1533782433.1533816989.1533822858.9; __utmz=94650624.1533822858.9.7.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/; WM_NI=S5gViyNVs14K%2BZoVerGK69gLlmtnH5NqzyHcCUY%2BiWm2ZaHATeI1gfsEnK%2BQ1jyP%2FROzbzDV0AyJHR4YQfBetXSRipyrYCFn%2BNdA%2FA8Mv80riS3cuMVJi%2BAFgCpXTiHBNHE%3D; WM_NIKE=9ca17ae2e6ffcda170e2e6ee84b674afedfbd3cd7d98b8e1d0f554f888a4abc76990b184badc4f89e7af8ece2af0fea7c3b92a91eba9b7ec738e8abdd2b741e986a1b7e87a8595fadae648b0b3bc8fcb3f8eafb69acb69818b97ccec5dafee9682cb4b98bb87d2e66eb19ba2acaa5bf3b6b7b1ae5a8da6ae9bc75ef49fb7abcb5af8879f87c16fb8889db3ec7cbbae97a4c566e992aca2ae4bfc93bad9b37aab8dfd84f8479696a7ccc44ea59dc0b9d7638c9e82a9c837e2a3; JSESSIONID-WYYY=sHwCKYJYxz6ODfURChA471BMF%5CSVf3%5CTc8Qcy9h9Whj6CfMxw4YWTMV7CIx5g6rqW8OBv04YGHwwq%2B%5CD1N61qknTP%2Fym%2BHJZ1ylSH1EabbQASc9ywIT8YvOr%2FpMgvmm1cbr2%2Bd6ssMYXuTlpOIrKqp%5C%2FM611EhmfAfU47%5CSQWAs%2BYzgY%3A1533828139236'
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0',
'Cookie': '_iuqxldmzr_=32; _ntes_nnid=8d4ef0883a3bcc9d3a2889b0bf36766a,1533782432391; _ntes_nuid=8d4ef0883a3bcc9d3a2889b0bf36766a; __utmc=94650624; WM_TID=GzmBlbRkRGQXeQiYuDVCfoEatU6VSsKC; playerid=19729878; __utma=94650624.1180067615.1533782433.1533816989.1533822858.9; __utmz=94650624.1533822858.9.7.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/; WM_NI=S5gViyNVs14K%2BZoVerGK69gLlmtnH5NqzyHcCUY%2BiWm2ZaHATeI1gfsEnK%2BQ1jyP%2FROzbzDV0AyJHR4YQfBetXSRipyrYCFn%2BNdA%2FA8Mv80riS3cuMVJi%2BAFgCpXTiHBNHE%3D; WM_NIKE=9ca17ae2e6ffcda170e2e6ee84b674afedfbd3cd7d98b8e1d0f554f888a4abc76990b184badc4f89e7af8ece2af0fea7c3b92a91eba9b7ec738e8abdd2b741e986a1b7e87a8595fadae648b0b3bc8fcb3f8eafb69acb69818b97ccec5dafee9682cb4b98bb87d2e66eb19ba2acaa5bf3b6b7b1ae5a8da6ae9bc75ef49fb7abcb5af8879f87c16fb8889db3ec7cbbae97a4c566e992aca2ae4bfc93bad9b37aab8dfd84f8479696a7ccc44ea59dc0b9d7638c9e82a9c837e2a3; JSESSIONID-WYYY=sHwCKYJYxz6ODfURChA471BMF%5CSVf3%5CTc8Qcy9h9Whj6CfMxw4YWTMV7CIx5g6rqW8OBv04YGHwwq%2B%5CD1N61qknTP%2Fym%2BHJZ1ylSH1EabbQASc9ywIT8YvOr%2FpMgvmm1cbr2%2Bd6ssMYXuTlpOIrKqp%5C%2FM611EhmfAfU47%5CSQWAs%2BYzgY%3A1533828139236'
}
def __get_songs(self,name):
def __get_songs(self, name):
d = '{"hlpretag":"<span class=\\"s-fc7\\">","hlposttag":"</span>","s":"%s","type":"1","offset":"0","total":"true","limit":"30","csrf_token":""}' % name
wyy = decrypt_music(d)
data = wyy.get_data()
url = 'https://music.163.com/weapi/cloudsearch/get/web?csrf_token='
response = requests.post(url,data= data,headers =self.headers).json()
response = requests.post(url, data=data, headers=self.headers).json()
return response['result']
def __get_mp3(self,id):
def __get_mp3(self, id):
d = '{"ids":"[%s]","br":320000,"csrf_token":""}' % id
wyy = decrypt_music(d)
data = wyy.get_data()
@@ -80,15 +78,14 @@ class Spider(object):
print(response)
return response['data'][0]['url']
def __download_mp3(self,url,filename):
def __download_mp3(self, url, filename):
abspath = os.path.abspath('.') # 获取绝对路径
os.chdir(abspath)
response = requests.get(url, headers=self.headers).content
path = os.path.join(abspath, filename)
with open(filename + '.mp3', 'wb') as f:
f.write(response)
print('下载完毕,可以在%s 路径下查看' % path + '.mp3')
print('下载完毕,可以在%s 路径下查看' % path + '.mp3')
def __print_info(self, songs):
"""打印歌曲需要下载的歌曲信息"""
@@ -98,7 +95,6 @@ class Spider(object):
songs_list.append((song['name'], song['id']))
return songs_list
def run(self):
while True:
name = input('请输入你需要下载的歌曲：')
@@ -119,9 +115,11 @@ class Spider(object):
break
print('程序结束！')
def main():
spider = Spider()
spider.run()
if __name__ == '__main__':
main()
main()
baidu/login_direct.py
+6 -5
View File
@@ -6,7 +6,6 @@ import execjs
import rsa
import base64
js_path = 'login.js'
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
@@ -37,7 +36,7 @@ def get_callback():
def _get_curtime():
return int(time.time()*1000)
return int(time.time() * 1000)
# 抓包也不是百分百可靠啊,这里?getapi一定要挨着https://passport.baidu.com/v2/api/写，才会到正确的路由
@@ -53,11 +52,12 @@ def get_token(gid, callback):
'logintype': 'basicLogin',
'callback': callback
}
headers.update(dict(Referer='http://pan.baidu.com/', Accept='*/*', Connection='keep-alive', Host='passport.baidu.com'))
headers.update(
dict(Referer='http://pan.baidu.com/', Accept='*/*', Connection='keep-alive', Host='passport.baidu.com'))
resp = session.get(url='https://passport.baidu.com/v2/api/?getapi', params=get_data, headers=headers)
if resp.status_code == 200 and callback in resp.text:
# 如果json字符串中带有单引号，会解析出错，只有统一成双引号才可以正确的解析
#data = eval(re.search(r'.*?\((.*)\)', resp.text).group(1))
# data = eval(re.search(r'.*?\((.*)\)', resp.text).group(1))
data = json.loads(re.search(r'.*?\((.*)\)', resp.text).group(1).replace("'", '"'))
return data.get('data').get('token')
else:
@@ -141,7 +141,7 @@ def login(token, gid, callback, rsakey, username, password):
's_K__oJTLlOSF9oXyne-LDQIAHcvLmJSM2JnXkMKDzpHOns2dwvap9obnlOeQ_43pCAIAINzY4uIKCgpdwK7Lv9uyw'
'ar1heSX5LvXuN-22IfhjvyRDQIAHcvLkGx0IGEvaDp7Nmk2ZjVlOg5RDn4fbB9oB3URDQIAHcvLrOnxpeSq7b_-s'
'-yz47Dgv4vUi_ua6ZrtgvCUDQIAHcvLS7au-rv1suCh7LPsvO-_4NSL1KTFtsWy3a_L',
'callback': 'parent.'+callback
'callback': 'parent.' + callback
}
resp = session.post(url='https://passport.baidu.com/v2/api/?login', data=post_data, headers=headers)
if 'err_no=0' in resp.text:
@@ -149,6 +149,7 @@ def login(token, gid, callback, rsakey, username, password):
else:
print('登录失败')
if __name__ == '__main__':
name = input('请输入用户名:\n')
passwd = input('请输入密码:\n')
baidu2/baidu.py
+3 -3
View File
@@ -14,7 +14,6 @@ import rsa
import base64
import time
"""
1. 百度的模拟登录，请确保你的账号能在浏览器下登录不需要验证码
百度的三次登录错误后，才会出现验证码，但是还有一种情况是百度的账号异常
@@ -48,7 +47,7 @@ def get_gid():
gid = list(gid)
for xy in range(len(gid)):
if gid[xy] in "xy":
r = int(random.random()*16)
r = int(random.random() * 16)
if gid[xy] == "x":
gid[xy] = hex(r).replace("0x", '').upper()
else:
@@ -121,6 +120,7 @@ def get_token():
# print(token_content_all)
return token_content_all['data']['token']
token = (get_token())
@@ -189,12 +189,12 @@ def login(username, password, key):
# f.write(html_index.content)
# f.close()
try:
input = raw_input
except:
pass
if __name__ == "__main__":
username = input("请输入你的手机号或者邮箱\n >:")
secret = input("请输入你的密码\n >:")
bilibili/bilibili.py
+4 -2
View File
@@ -16,11 +16,13 @@ github:https://github.com/CriseLYJ/
update_time:2019-3-7
"""
class BiliBili():
"""
登陆B站, 处理极验验证码
电脑的缩放比例需要为100%, 否则验证码图片的获取会出现问题
"""
def __init__(self, username, password):
"""
初始化
@@ -128,7 +130,7 @@ class BiliBili():
:param img2: 缺块验证码
:return: 第二个缺块的左侧的x坐标
"""
left = 60 # 大致忽略掉第一个缺块
left = 60 # 大致忽略掉第一个缺块
for i in range(left, img1.size[0]):
for j in range(img1.size[1]):
if not self.is_pixel_equal(img1, img2, i, j):
@@ -214,5 +216,5 @@ if __name__ == '__main__':
ACCOUNT = input('请输入您的账号:')
PASSOWRD = input('请输入您的密码:')
test = BiliBili(ACCOUNT, PASSOWRD) # 输入账号和密码
test = BiliBili(ACCOUNT, PASSOWRD) # 输入账号和密码
test.crack()
csdn/csdn_login.py
+2 -2
View File
@@ -3,7 +3,6 @@
import requests
from bs4 import BeautifulSoup
# 获取登陆页面
session = requests.session()
url = 'https://passport.csdn.net/account/login?from=http://my.csdn.net/my/mycsdn'
@@ -39,6 +38,7 @@ def login(post_data):
session.post(url, data=post_data, headers=get_post_headers())
return session
if __name__ == '__main__':
session = login(get_post_data())
# 检查是否正常登录
@@ -51,4 +51,4 @@ if __name__ == '__main__':
'Accept': 'Accept'
}
resp = session.get(home_page, headers='')
print(resp.text)
print(resp.text)
douban/douban.py
+6 -4
View File
@@ -12,6 +12,7 @@ from urllib.request import urlretrieve
import requests
from bs4 import BeautifulSoup
from os import remove
try:
import cookielib
except:
@@ -26,12 +27,12 @@ url = 'https://accounts.douban.com/login'
datas = {'source': 'index_nav',
'remember': 'on'}
headers = {'Host':'www.douban.com',
headers = {'Host': 'www.douban.com',
'Referer': 'https://www.douban.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding':'gzip, deflate, br'}
'Accept-Encoding': 'gzip, deflate, br'}
# 尝试使用cookie信息
session = requests.session()
@@ -40,7 +41,7 @@ try:
session.cookies.load(ignore_discard=True)
except:
print("Cookies未能加载")
#cookies加载不成功，则输入账号密码信息
# cookies加载不成功，则输入账号密码信息
datas['form_email'] = input('Please input your account:')
datas['form_password'] = input('Please input your password:')
@@ -91,13 +92,14 @@ def login():
page = login_page.text
soup = BeautifulSoup(page, "html.parser")
result = soup.findAll('div', attrs={'class': 'title'})
#进入豆瓣登陆后页面，打印热门内容
# 进入豆瓣登陆后页面，打印热门内容
for item in result:
print(item.find('a').get_text())
# 保存 cookies 到文件，
# 下次可以使用 cookie 直接登录，不需要输入账号和密码
session.cookies.save()
if __name__ == '__main__':
if isLogin():
print('Login successfully')
facebook/facebook.py
+8 -7
View File
@@ -4,8 +4,8 @@ import argparse
import requests
import pyquery
def login(session, email, password):
'''
Attempt to login to Facebook. Returns user ID, xs token and
fb_dtsg token. All 3 are required to make requests to
@@ -15,28 +15,29 @@ def login(session, email, password):
# Navigate to Facebook's homepage to load Facebook's cookies.
response = session.get('https://m.facebook.com')
# Attempt to login to Facebook
response = session.post('https://m.facebook.com/login.php', data={
'email': email,
'pass': password
}, allow_redirects=False)
# If c_user cookie is present, login was successful
if 'c_user' in response.cookies:
# Make a request to homepage to get fb_dtsg token
homepage_resp = session.get('https://m.facebook.com/home.php')
dom = pyquery.PyQuery(homepage_resp.text.encode('utf8'))
fb_dtsg = dom('input[name="fb_dtsg"]').val()
return fb_dtsg, response.cookies['c_user'], response.cookies['xs']
else:
return False
return False
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Login to Facebook')
parser.add_argument('email', help='Email address')
parser.add_argument('password', help='Login password')
@@ -49,7 +50,7 @@ if __name__ == "__main__":
})
fb_dtsg, user_id, xs = login(session, args.email, args.password)
if user_id:
print('{0}:{1}:{2}'.format(fb_dtsg, user_id, xs))
else:
guoke/guoke.py
+18 -20
View File
@@ -1,16 +1,15 @@
import requests
import re
headers_login = {
'Accept-Language':'zh-CN,zh;q=0.8',
'Cache-Control':'no-cache',
'Connection':'keep-alive',
'Host':'account.guokr.com',
'Pragma':'no-cache',
'Cookie':'__utmt=1; __utma=253067679.2102330349.1540780238.1540780238.1541122809.2; __utmb=253067679.12.9.1541122812936; __utmc=253067679; __utmz=253067679.1540780238.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmv=253067679.|1=Is%20Registered=No=1; session=afcf1b0f-c71b-43d2-8046-f60ae28f9b45',
'Referer':'https://account.guokr.com/sign_in/',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.61 Safari/537.36'
'Accept-Language': 'zh-CN,zh;q=0.8',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Host': 'account.guokr.com',
'Pragma': 'no-cache',
'Cookie': '__utmt=1; __utma=253067679.2102330349.1540780238.1540780238.1541122809.2; __utmb=253067679.12.9.1541122812936; __utmc=253067679; __utmz=253067679.1540780238.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmv=253067679.|1=Is%20Registered=No=1; session=afcf1b0f-c71b-43d2-8046-f60ae28f9b45',
'Referer': 'https://account.guokr.com/sign_in/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.61 Safari/537.36'
}
session = requests.Session()
@@ -29,25 +28,24 @@ username = input('请输入用户名：')
password = input('请输入密码：')
captcha = input('请输入验证码 : ')
data = {
'csrf_token':csrf_token,
'username':username,
'password':password,
'captcha':captcha,
'captcha_rand':captcha_rand,
'permanent':'y ' ,
'csrf_token': csrf_token,
'username': username,
'password': password,
'captcha': captcha,
'captcha_rand': captcha_rand,
'permanent': 'y ',
}
response = session.post(url, data=data)
with open('response.html', 'w', encoding='utf-8') as fw:
fw.write(response.text)
#print(response.cookies)
#print(session.cookies)
# print(response.cookies)
# print(session.cookies)
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.61 Safari/537.36'
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.61 Safari/537.36'
}
homepage = 'https://www.guokr.com/i/0210199872/'
with open('homepage.html', 'w', encoding='utf-8') as fw:
guoke/guoke_spider.py
+22 -22
View File
@@ -17,6 +17,7 @@ update_time:2019-3-7
"""
def get_index(offset):
base_url = 'http://www.guokr.com/apis/minisite/article.json?'
data = {
@@ -25,7 +26,7 @@ def get_index(offset):
'offset': offset
}
url = base_url + urlencode(data)
#print(url)
# print(url)
try:
resp = requests.get(url)
if codes.ok == resp.status_code:
@@ -33,7 +34,8 @@ def get_index(offset):
except requests.ConnectionError:
return None
#解析出文章的url
# 解析出文章的url
def get_url(json):
if json.get('result'):
result = json.get('result')
@@ -50,27 +52,26 @@ def get_url(json):
"""
#解析文章详情页
# 解析文章详情页
def get_text(url):
html=requests.get(url).text
html = requests.get(url).text
print(html)
soup=bsp(html,'lxml')
title=soup.find('h1',id='articleTitle').get_text()
autor=soup.find('div',class_="content-th-info").find('a').get_text()
article_content=soup.find('div',class_="document").find_all('p')
all_p = [i.get_text() for i in article_content if not i.find('img') and not i.find('a')]#去除标签
soup = bsp(html, 'lxml')
title = soup.find('h1', id='articleTitle').get_text()
autor = soup.find('div', class_="content-th-info").find('a').get_text()
article_content = soup.find('div', class_="document").find_all('p')
all_p = [i.get_text() for i in article_content if not i.find('img') and not i.find('a')] # 去除标签
article = '\n'.join(all_p)
yield {"title":title,"autor":autor,"article":article}
yield {"title": title, "autor": autor, "article": article}
def save_article(content):
try:
if content.get('title'):
file_name=str(content.get('title'))+'.txt'
with open(file_name, 'w',encoding='utf-8') as f:
#f.write(json.dumps(content,ensure_ascii=False))
f.write('\n'.join([str(content.get('title')),str(content.get('autor')),str(content.get('article'))]))
file_name = str(content.get('title')) + '.txt'
with open(file_name, 'w', encoding='utf-8') as f:
# f.write(json.dumps(content,ensure_ascii=False))
f.write('\n'.join([str(content.get('title')), str(content.get('autor')), str(content.get('article'))]))
print('Downloaded article path is %s' % file_name)
else:
print('Already Downloaded', file_name)
@@ -78,14 +79,13 @@ def save_article(content):
print('Failed to Save Image，item %s' % content)
def main(offset):
result=get_index(offset)
all_url=get_url(result)
result = get_index(offset)
all_url = get_url(result)
for url in all_url:
article=get_text(url)
article = get_text(url)
for art in article:
#print(art)
# print(art)
save_article(art)
@@ -93,6 +93,6 @@ GROUP_START = 0
GROUP_END = 7
if __name__ == '__main__':
for i in range(GROUP_START,GROUP_END+1):
main(offset=i*20+18)
for i in range(GROUP_START, GROUP_END + 1):
main(offset=i * 20 + 18)
time.sleep(1)
jd_login/login_by_selenium.py
+1 -2
View File
@@ -2,7 +2,6 @@
import time
from selenium import webdriver
# 强东的系统登陆模块没有用iframe,定位也没有微博那么复杂，可以靠id定位登陆
login_url = 'https://passport.jd.com/new/login.aspx'
driver = webdriver.PhantomJS()
@@ -23,4 +22,4 @@ time.sleep(5)
# cookie和前面一样的方式获取和保存
cookies = driver.get_cookies()
driver.close()
driver.close()
lagou/Lagou.py
+7 -11
View File
@@ -9,7 +9,6 @@ import requests
import hashlib
from bs4 import BeautifulSoup
"""
info:
author:CriseLYJ
@@ -28,21 +27,20 @@ class Lagou_login(object):
' Core/1.53.4882.400 QQBrowser/9.7.13059.400',
'X-Requested-With': 'XMLHttpRequest'}
#密码加密
# 密码加密
def encryptPwd(self, passwd):
# 对密码进行了md5双重加密
passwd = hashlib.md5(passwd.encode('utf-8')).hexdigest()
# veennike 这个值是在js文件找到的一个写死的值
passwd = 'veenike'+passwd+'veenike'
passwd = 'veenike' + passwd + 'veenike'
passwd = hashlib.md5(passwd.encode('utf-8')).hexdigest()
return passwd
#获取请求token
# 获取请求token
def getTokenCode(self):
login_page = 'https://passport.lagou.com/login/login.html'
data = self.session.get(login_page, headers= self.HEADERS)
data = self.session.get(login_page, headers=self.HEADERS)
soup = BeautifulSoup(data.content, "lxml", from_encoding='utf-8')
'''
@@ -63,7 +61,6 @@ class Lagou_login(object):
return anti_token
# 人工读取验证码并返回
def getCaptcha(self):
captchaImgUrl = 'https://passport.lagou.com/vcode/create?from=register&refresh=%s' % time.time()
@@ -84,19 +81,18 @@ class Lagou_login(object):
print('你输入的验证码是:% s' % captcha)
return captcha
# 登陆操作
def login(self, user, passwd, captchaData=None, token_code=None):
postData = {'isValidate': 'true',
'password': passwd,
# 如需验证码,则添加上验证码
'request_form_verifyCode': (captchaData if captchaData!=None else ''),
'request_form_verifyCode': (captchaData if captchaData != None else ''),
'submit': '',
'username': user
}
login_url = 'https://passport.lagou.com/login/login.json'
#头信息添加tokena
# 头信息添加tokena
login_headers = self.HEADERS.copy()
token_code = self.getTokenCode() if token_code is None else token_code
login_headers.update(token_code)
@@ -110,7 +106,7 @@ class Lagou_login(object):
elif data['state'] == 10010:
print(data['message'])
captchaData = self.getCaptcha()
token_code = {'X-Anit-Forge-Code' : data['submitCode'], 'X-Anit-Forge-Token' : data['submitToken']}
token_code = {'X-Anit-Forge-Code': data['submitCode'], 'X-Anit-Forge-Token': data['submitToken']}
return self.login(user, passwd, captchaData, token_code)
else:
print(data['message'])
qqzone/qq_zone.py
+9 -11
View File
@@ -7,10 +7,10 @@ github:https://github.com/CriseLYJ/
update_time:2019-3-7
"""
import time #用来延时
import time # 用来延时
from selenium import webdriver
driver = webdriver.Chrome() #选择浏览器，此处我选择的Chrome
driver = webdriver.Chrome() # 选择浏览器，此处我选择的Chrome
QQ_NUMBER = input('请输入你的QQ号')
PASSWORD = input('请输入你的QQ密码')
@@ -19,21 +19,19 @@ driver.switch_to.frame('login_frame')
driver.find_element_by_id('switcher_plogin').click()
driver.find_element_by_name('u').clear()
driver.find_element_by_name('u').send_keys(QQ_NUMBER) # 此处输入你的QQ号
driver.find_element_by_name('u').send_keys(QQ_NUMBER) # 此处输入你的QQ号
driver.find_element_by_name('p').clear()
driver.find_element_by_name('p').send_keys(PASSWORD) # 此处输入你的QQ密码
driver.find_element_by_name('p').send_keys(PASSWORD) # 此处输入你的QQ密码
driver.execute_script("document.getElementById('login_button').parentNode.hidefocus=false;")
driver.find_element_by_xpath('//*[@id="loginform"]/div[4]/a').click()
driver.find_element_by_id('login_button').click()
driver.find_element_by_id('login_button').click()
time.sleep(10) # 因为我曾经是QQ会员，所以每次登陆时都会提醒我要不要再续费的弹窗...
time.sleep(10) # 因为我曾经是QQ会员，所以每次登陆时都会提醒我要不要再续费的弹窗...
driver.find_element_by_id('dialog_button_1').click() # 这个地方是我把那个弹窗给点击了，配合上面的延时用的，延时是等待那个弹窗出现，然后此处点击取消
driver.find_element_by_id('dialog_button_1').click() # 这个地方是我把那个弹窗给点击了，配合上面的延时用的，延时是等待那个弹窗出现，然后此处点击取消
btns = driver.find_elements_by_css_selector('a.item.qz_like_btn_v3') # 此处是CSS选择器
btns = driver.find_elements_by_css_selector('a.item.qz_like_btn_v3') # 此处是CSS选择器
for btn in btns:
btn.click()
btn.click()
sina/sina.py
+2 -3
View File
@@ -11,9 +11,8 @@ update_time:2019-3-7
def get_login(phone, pwd):
new_time = str(int(time.time()))
sign = new_time+'_'+hashlib.md5((phone + pwd + new_time).encode("utf-8")).hexdigest()
sign = new_time + '_' + hashlib.md5((phone + pwd + new_time).encode("utf-8")).hexdigest()
print(sign)
url = "https://appblog.sina.com.cn/api/passport/v3_1/login.php"
@@ -39,4 +38,4 @@ if __name__ == '__main__':
phone = input("你输入你的账号:")
pwd = input("请输入你的密码：")
get_login(phone, pwd)
get_login(phone, pwd)
sina/spider/Ajax_weibo.py
+10 -13
View File
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
from urllib.parse import urlencode
import requests,pymysql
import requests, pymysql
from pyquery import PyQuery as pq
from selenium import webdriver
from time import sleep
@@ -18,7 +18,7 @@ sql = "USE python;"
cursor.execute(sql)
connection.commit()
base_url='https://m.weibo.cn/api/container/getIndex?'
base_url = 'https://m.weibo.cn/api/container/getIndex?'
headers = {
'Host': 'm.weibo.cn',
'Referer': 'https://m.weibo.cn/u/2145291155',
@@ -26,6 +26,7 @@ headers = {
'X-Requested-With': 'XMLHttpRequest',
}
def create_sheet(bozhu):
try:
weibo = '''
@@ -43,6 +44,7 @@ def create_sheet(bozhu):
except:
pass
def url_get():
# # 自动保持cookie,不需要自己维护cookie内容
# cookies = {}
@@ -61,15 +63,15 @@ def url_get():
# }
# # get请求，应答解码
# response = s.get(url=xl_url, headers=header,cookies=cookies)
browser=webdriver.PhantomJS()
browser = webdriver.PhantomJS()
browser.get(url='https://m.weibo.cn/')
wb_name=browser.find_element_by_class_name("W_input")
wb_name = browser.find_element_by_class_name("W_input")
wb_name.send_keys(input('输入博主ID：'))
sleep(10)
search=browser.find_element_by_class_name('W_ficon ficon_search S_ficon')
search = browser.find_element_by_class_name('W_ficon ficon_search S_ficon')
search.click()
sleep(5)
bz_num=browser.find_element_by_class_name('name_txt')
bz_num = browser.find_element_by_class_name('name_txt')
bz_num.click()
sleep(5)
# 开启了一个新页面，需要跳转到新页面
@@ -77,8 +79,6 @@ def url_get():
browser.switch_to_window(handles[1])
# https://m.weibo.cn/api/container/getIndex?type=uid&value=2145291155&containerid=1076032145291155
# 拼接url
def get_page(page):
@@ -104,7 +104,7 @@ def get_page(page):
def parse_page(json):
if json:
items = json.get('data').get('cards')
for index,item in enumerate(items):
for index, item in enumerate(items):
if page == 1 and index == 1:
continue
else:
@@ -139,10 +139,7 @@ if __name__ == '__main__':
for result in results:
print(result)
cursor.close()
# 可以爬任意指定博主所有微博，以博主名建立表，分别储存信息
# 使用selenium+PhantomJS抓取对应博主主页链接
# 使用selenium+PhantomJS抓取对应博主主页链接
taobao/taobao.py
+18 -16
View File
@@ -1,4 +1,4 @@
#！/usr/bin/env python
# ！/usr/bin/env python
# -*- coding:utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.by import By
@@ -11,34 +11,35 @@ import time
info:
author:CriseLYJ
github:https://github.com/CriseLYJ/
update_time:2019-3-7
update_time:2019-3-8
"""
class loginTB(object):
def __init__(self):
self.driver = webdriver.Chrome()
self.driver.maximize_window()
# 设置一个智能等待
self.wait = WebDriverWait(self.driver,5)
self.wait = WebDriverWait(self.driver, 5)
def login(self,key,pw):
def login(self, key, pw):
url = 'https://login.taobao.com/member/login.jhtml'
self.driver.get(url)
try:
# 寻找密码登陆按钮
login_links = self.wait.until(
EC.presence_of_element_located((By.XPATH,"//a[text()='密码登录']"))
EC.presence_of_element_located((By.XPATH, "//a[text()='密码登录']"))
)
login_links.click()
except TimeoutException as e:
print("找不到登陆入口，原因是：",e)
print("找不到登陆入口，原因是：", e)
else:
# 输入账号密码
input_key = self.wait.until(
EC.presence_of_element_located((By.XPATH,"//input[@name='TPL_username']"))
EC.presence_of_element_located((By.XPATH, "//input[@name='TPL_username']"))
)
input_pw = self.wait.until(
EC.presence_of_element_located((By.XPATH,"//input[@name='TPL_password']"))
EC.presence_of_element_located((By.XPATH, "//input[@name='TPL_password']"))
)
input_key.clear()
input_pw.clear()
@@ -48,7 +49,7 @@ class loginTB(object):
try:
# 试探能否找到个人信息，如果找不到说明登录失败
user_info = self.wait.until(
EC.presence_of_element_located((By.XPATH,"//div[@class='m-userinfo']"))
EC.presence_of_element_located((By.XPATH, "//div[@class='m-userinfo']"))
)
print('已经登陆成功，进入了个人中心')
except TimeoutException:
@@ -59,7 +60,7 @@ class loginTB(object):
try:
# 尝试找手机验证框，如果能找到说明要手机验证
frame = self.wait.until(
EC.presence_of_element_located((By.XPATH,'//div[@class="login-check-left"]/iframe'))
EC.presence_of_element_located((By.XPATH, '//div[@class="login-check-left"]/iframe'))
)
print('本次登录需要进行手机验证...')
except TimeoutException:
@@ -67,20 +68,20 @@ class loginTB(object):
print('登录失败，目测是账号或密码有误，请检查后重新登录...')
key = input('请重新输入账号：').strip()
pw = input('请重新输入密码：').strip()
self.login(key,pw)
self.login(key, pw)
else:
self.driver.switch_to.frame(frame)
phone_num = self.wait.until(
EC.presence_of_element_located((By.XPATH,'//button[@id="J_GetCode"]'))
EC.presence_of_element_located((By.XPATH, '//button[@id="J_GetCode"]'))
)
phone_num.click()
phone_key = input('请输入手机验证码：').strip()
key_send = self.wait.until(
EC.presence_of_element_located((By.XPATH,'//input[@id="J_Phone_Checkcode"]'))
EC.presence_of_element_located((By.XPATH, '//input[@id="J_Phone_Checkcode"]'))
)
key_send.send_keys(phone_key)
go_button = self.wait.until(
EC.presence_of_element_located((By.XPATH,'//input[@type="submit"]'))
EC.presence_of_element_located((By.XPATH, '//input[@type="submit"]'))
)
go_button.click()
user_info = self.wait.until(
@@ -88,8 +89,9 @@ class loginTB(object):
)
print('手机验证登陆成功！！！')
if __name__ == '__main__':
t = time.time()
l = loginTB()
l.login('username','password')
print('登录完成，耗时{:.2f}秒'.format(float(time.time()-t)))
l.login('username', 'password')
print('登录完成，耗时{:.2f}秒'.format(float(time.time() - t)))
tuchong/tuchong.py
+10 -7
View File
@@ -1,6 +1,4 @@
import requests
import json
import time
import re
import os
from hashlib import md5
@@ -23,6 +21,7 @@ headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
# 获取imageID
def get_imageID(term, page):
try:
@@ -35,6 +34,7 @@ def get_imageID(term, page):
except ConnectionError:
return None
# 解析imageID里面的图片id
def parse_imgID(imageID):
print('解析imageID')
@@ -60,20 +60,22 @@ def get_ImageJPG(id):
except ConnectionError:
return None
# 解析html里面的图片url
def parse_imgURL(html):
if html:
print('解析HTML图片URL...')
url = re.findall('<div.*?class="image-cover".*?<img.*?src="(.*?)">.*?</div>', html, re.S)
#url = re.findall('<title>(.*?)</title>', html, re.S)
url = re.findall('<div.*?class="image-cover".*?<img.*?src="(.*?)">.*?</div>', html, re.S)
# url = re.findall('<title>(.*?)</title>', html, re.S)
for item in url:
print("准备下载...", item)
download_image(item)
return None
def download_image(url):
try:
urls = 'https:' + url
urls = 'https:' + url
ir = requests.get(urls)
if ir.status_code == 200:
save_image(ir.content)
@@ -81,6 +83,7 @@ def download_image(url):
except RequestException:
return None
def save_image(content):
file_path = '{0}/{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')
if not os.path.exists(file_path):
@@ -89,12 +92,12 @@ def save_image(content):
f.close()
print('下载成功----------------------')
def main():
term = input('输入想要搜索的内容: ')
for i in range(1, 7):
get_imageID(term, i)
if __name__ == '__main__':
main()
main()
twitter/setup.py
+1 -1
View File
@@ -101,4 +101,4 @@ setup(
cmdclass={
'upload': UploadCommand,
},
)
)
twitter/twitter_scraper.py
+14 -12
View File
@@ -36,17 +36,19 @@ def get_tweets(user, pages=25):
tweetId = tweet.find(
'.js-permalink')[0].attrs['data-conversation-id']
time = datetime.fromtimestamp(
int(tweet.find('._timestamp')[0].attrs['data-time-ms'])/1000.0)
int(tweet.find('._timestamp')[0].attrs['data-time-ms']) / 1000.0)
interactions = [x.text for x in tweet.find(
'.ProfileTweet-actionCount')]
replies = int(interactions[0].split(" ")[0].replace(comma, "").replace(dot,""))
replies = int(interactions[0].split(" ")[0].replace(comma, "").replace(dot, ""))
retweets = int(interactions[1].split(" ")[
0].replace(comma, "").replace(dot,""))
likes = int(interactions[2].split(" ")[0].replace(comma, "").replace(dot,""))
0].replace(comma, "").replace(dot, ""))
likes = int(interactions[2].split(" ")[0].replace(comma, "").replace(dot, ""))
hashtags = [hashtag_node.full_text for hashtag_node in tweet.find('.twitter-hashtag')]
urls = [url_node.attrs['data-expanded-url'] for url_node in tweet.find('a.twitter-timeline-link:not(.u-hidden)')]
photos = [photo_node.attrs['data-image-url'] for photo_node in tweet.find('.AdaptiveMedia-photoContainer')]
urls = [url_node.attrs['data-expanded-url'] for url_node in
tweet.find('a.twitter-timeline-link:not(.u-hidden)')]
photos = [photo_node.attrs['data-image-url'] for photo_node in
tweet.find('.AdaptiveMedia-photoContainer')]
videos = []
video_nodes = tweet.find(".PlayableMedia-player")
for node in video_nodes:
@@ -57,11 +59,11 @@ def get_tweets(user, pages=25):
video_id = tmp[:tmp.index('.jpg')]
videos.append({'id': video_id})
tweets.append({'tweetId': tweetId, 'time': time, 'text': text,
'replies': replies, 'retweets': retweets, 'likes': likes,
'replies': replies, 'retweets': retweets, 'likes': likes,
'entries': {
'hashtags': hashtags, 'urls': urls,
'photos': photos, 'videos': videos
}
'hashtags': hashtags, 'urls': urls,
'photos': photos, 'videos': videos
}
})
last_tweet = html.find('.stream-item')[-1].attrs['data-item-id']
@@ -72,7 +74,7 @@ def get_tweets(user, pages=25):
yield tweet
r = session.get(
url, params = {'max_position': last_tweet}, headers = headers)
url, params={'max_position': last_tweet}, headers=headers)
pages += -1
yield from gen_tweets(pages)
webWeixin/webWeixin.py
+17 -18
View File
@@ -16,7 +16,7 @@ update_time:2019-3-6
session = requests.session()
headers = {
'User-Agent' : 'Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0'
'User-Agent': 'Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0'
}
QRImgPath = os.path.split(os.path.realpath(__file__))[0] + os.sep + 'webWeixinQr.jpg'
@@ -36,7 +36,7 @@ SyncKey = ''
def getUUID():
global uuid,session
global uuid, session
url = 'https://login.weixin.qq.com/jslogin'
params = {
@@ -176,10 +176,9 @@ def login():
def webwxinit():
url = base_uri + \
'/webwxinit?pass_ticket=%s&skey=%s&r=%s' % (
pass_ticket, skey, int(time.time()))
'/webwxinit?pass_ticket=%s&skey=%s&r=%s' % (
pass_ticket, skey, int(time.time()))
params = {
'BaseRequest': BaseRequest
}
@@ -188,7 +187,7 @@ def webwxinit():
h['ContentType'] = 'application/json; charset=UTF-8'
response = session.post(url, data=json.dumps(params), headers=h)
data = response.content.decode('utf-8')
#print(data)
# print(data)
global ContactList, My, SyncKey
@@ -211,10 +210,9 @@ def webwxinit():
def webwxgetcontact():
url = base_uri + \
'/webwxgetcontact?pass_ticket=%s&skey=%s&r=%s' % (
pass_ticket, skey, int(time.time()))
'/webwxgetcontact?pass_ticket=%s&skey=%s&r=%s' % (
pass_ticket, skey, int(time.time()))
h = headers
h['ContentType'] = 'application/json; charset=UTF-8'
@@ -226,8 +224,12 @@ def webwxgetcontact():
MemberList = dic['MemberList']
# 倒序遍历,不然删除的时候出问题..
SpecialUsers = ["newsapp", "fmessage", "filehelper", "weibo", "qqmail", "tmessage", "qmessage", "qqsync", "floatbottle", "lbsapp", "shakeapp", "medianote", "qqfriend", "readerapp", "blogapp", "facebookapp", "masssendapp",
"meishiapp", "feedsapp", "voip", "blogappweixin", "weixin", "brandsessionholder", "weixinreminder", "wxid_novlwrv3lqwv11", "gh_22b87fa7cb3c", "officialaccounts", "notification_messages", "wxitil", "userexperience_alarm"]
SpecialUsers = ["newsapp", "fmessage", "filehelper", "weibo", "qqmail", "tmessage", "qmessage", "qqsync",
"floatbottle", "lbsapp", "shakeapp", "medianote", "qqfriend", "readerapp", "blogapp", "facebookapp",
"masssendapp",
"meishiapp", "feedsapp", "voip", "blogappweixin", "weixin", "brandsessionholder", "weixinreminder",
"wxid_novlwrv3lqwv11", "gh_22b87fa7cb3c", "officialaccounts", "notification_messages", "wxitil",
"userexperience_alarm"]
for i in range(len(MemberList) - 1, -1, -1):
Member = MemberList[i]
if Member['VerifyFlag'] & 8 != 0: # 公众号/服务号
@@ -241,8 +243,8 @@ def webwxgetcontact():
return MemberList
def main():
def main():
if not getUUID():
print('获取uuid失败')
return
@@ -258,7 +260,7 @@ def main():
if not login():
print('登录失败')
return
#登录完成, 下面查询好友
# 登录完成, 下面查询好友
if not webwxinit():
print('初始化失败')
return
@@ -267,14 +269,11 @@ def main():
print('通讯录共%s位好友' % len(MemberList))
for x in MemberList :
for x in MemberList:
sex = '未知' if x['Sex'] == 0 else '男' if x['Sex'] == 1 else '女'
print('昵称:%s, 性别:%s, 备注:%s, 签名:%s' % (x['NickName'], sex, x['RemarkName'], x['Signature']))
if __name__ == '__main__':
print('开始')
main()
main()