网易云音乐网址为：https://music.163.com/
思路是进入后输入一个歌曲名，点击搜索按钮，通过开发者调试工具捕获搜索请求，捕获到的数据信息如下：
所有的歌曲相关信息都在result里面，每个a标签（超链接）里面携带着大量信息，包括歌曲名，id号，演唱者等等，这里最需要的最关键的一个信息就是它的id号，因为歌曲名作者可能会重复，但是id是唯一的，可以通过id来获取该歌曲其他信息。
再查看该请求的URL，https://music.163.com/weapi/cloudsearch/get/web?csrf_token= ，可以通过此URL来获取歌曲的id
获取到id后如何搜索呢， 以第一首歌曲为例，进入到详细页面：
发现网址后面只需要填写id号再做一些字符串的拼接即可
那这样一来就可以实现了，搜索歌曲的id：
import requests
from Netease_cloud_comment_capture.Encrypt import Encrypted
class search():
'''跟歌单直接下载的不同之处，1.就是headers的referer
2.加密的text内容不一样！
3.搜索的URL也是不一样的
输入搜索内容，可以获取歌曲ID
'''
def __init__(self):
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
'Host': 'music.163.com',
'Referer': 'http://music.163.com/search/'}
self.main_url='http://music.163.com/'
self.session = requests.Session()
self.session.headers=self.headers
self.ep = Encrypted()
def search_song(self, search_content, search_type=1, limit=9):
"""
根据音乐名搜索
:params search_content: 音乐名
:params search_type: 不知
:params limit: 返回结果数量
return: 可以得到id 再进去歌曲具体的url
"""
url = 'http://music.163.com/weapi/cloudsearch/get/web?csrf_token='
text = {'s': search_content, 'type': search_type, 'offset': 0, 'sub': 'false', 'limit': limit}
data = self.ep.search(text)
resp = self.session.post(url, data=data)
result = resp.json()
if result['result']['songCount']<= 0:
print('搜不到！！')
else:
songs = result['result']['songs']
for song in songs:
song_id = song['id']
return song_id 在歌曲详情页面里，可以继续利用调试工具抓取评论对应的数据包，可以发现要抓取的评论在data里面的comments放着
那这样就可以通过表单获取评论内容了
但是通过数据包头发现，表单的数据是加密的，params和encSecKey需要进行解码。
查了一下是AES加密，现在以及有很多解密方法了，那加密问题也破解了：
import base64
import json
import os
from binascii import hexlify
from Crypto.Cipher import AES
second_param = "010001"
third_param = "00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7"
forth_param = "0CoJUm6Qyw8W8jud"
# 加密解密
class Encrypted():
'''传入歌曲的ID，加密生成'params'、'encSecKey 返回'''
def __init__(self):
self.pub_key = second_param
self.modulus = third_param
self.nonce = forth_param
def create_secret_key(self, size):
return hexlify(os.urandom(size))[:16].decode('utf-8')
def aes_encrypt(self, text, key):
iv = '0102030405060708'
pad = 16 - len(text) % 16
text = text + pad * chr(pad)
encryptor = AES.new(key.encode('utf-8'), AES.MODE_CBC, iv.encode('utf-8'))
result = encryptor.encrypt(text.encode('utf-8'))
result_str = base64.b64encode(result).decode('utf-8')
return result_str
def rsa_encrpt(self, text, pubKey, modulus):
text = text[::-1]
rs = pow(int(hexlify(text.encode('utf-8')), 16), int(pubKey, 16), int(modulus, 16))
return format(rs, 'x').zfill(256)
def search(self, text):
text = json.dumps(text)
i = self.create_secret_key(16)
encText = self.aes_encrypt(text, self.nonce)
encText = self.aes_encrypt(encText, i)
encSecKey = self.rsa_encrpt(i, self.pub_key, self.modulus)
data = {'params': encText, 'encSecKey': encSecKey}
return dataimport requests, base64
from Crypto.Cipher import AES
from Netease_cloud_comment_capture.Encrypt import forth_param
def get_params(page): # page为传入页数
iv = "0102030405060708"
first_key = forth_param
second_key = 16 * 'F'
if(page == 1): # 如果为第一页
first_param = '{rid:"", offset:"0", total:"true", limit:"20", csrf_token:""}'
h_encText = AES_encrypt(first_param, first_key, iv)
else:
offset = str((page - 1) * 20)
first_param = '{rid:"", offset:"%s", total:"%s", limit:"20", csrf_token:""}' %(offset, 'false')
h_encText = AES_encrypt(first_param, first_key, iv)
h_encText = AES_encrypt(h_encText, second_key, iv)
return h_encText
def get_encSecKey():
encSecKey = "257348aecb5e556c066de214e531faadd1c55d814f9be95fd06d6bff9f4c7a41f831f6394d5a3fd2e3881736d94a02ca919d952872e7d0a50ebfa1769a7a62d512f5f1ca21aec60bc3819a9c3ffca5eca9a0dba6d6f7249b06f5965ecfff3695b54e1c28f3f624750ed39e7de08fc8493242e26dbc4484a01c76f739e135637c"
return encSecKey
def AES_encrypt(text, key, iv):
if type(text) == type(b'123'):
text = text.decode('utf-8')
# text=text.decode('utf-8')
pad = 16 - len(text) % 16
text = text + pad * chr(pad)
iv = iv.encode('utf-8')
key = key.encode('utf-8')
encryptor = AES.new((key), AES.MODE_CBC, (iv))
text = text.encode('utf-8')
encrypt_text = encryptor.encrypt(text)
encrypt_text = base64.b64encode(encrypt_text)
return encrypt_text
# 构造函数获取歌手信息
def get_comments_json(url, data):
headers={'Accept': 'text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, image/apng, */*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN, zh;q=0.9',
'Connection': 'keep-alive',
'Cookie': 'WM_TID=36fj4OhQ7NdU9DhsEbdKFbVmy9tNk1KM; _iuqxldmzr_=32; _ntes_nnid=26fc3120577a92f179a3743269d8d0d9, 1536048184013; _ntes_nuid=26fc3120577a92f179a3743269d8d0d9; __utmc=94650624; __utmz=94650624.1536199016.26.8.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); WM_NI=2Uy%2FbtqzhAuF6WR544z5u96yPa%2BfNHlrtTBCGhkg7oAHeZje7SJiXAoA5YNCbyP6gcJ5NYTs5IAJHQBjiFt561sfsS5Xg%2BvZx1OW9mPzJ49pU7Voono9gXq9H0RpP5HTclE%3D; WM_NIKE=9ca17ae2e6ffcda170e2e6eed5cb8085b2ab83ee7b87ac8c87cb60f78da2dac5439b9ca4b1d621f3e900b4b82af0fea7c3b92af28bb7d0e180b3a6a8a2f84ef6899ed6b740baebbbdab57394bfe587cd44b0aebcb5c14985b8a588b6658398abbbe96ff58d868adb4bad9ffbbacd49a2a7a0d7e6698aeb82bad779f7978fabcb5b82b6a7a7f73ff6efbd87f259f788a9ccf552bcef81b8bc6794a686d5bc7c97e99a90ee66ade7a9b9f4338cf09e91d33f8c8cad8dc837e2a3; JSESSIONID-WYYY=G%5CSvabx1X1F0JTg8HK5Z%2BIATVQdgwh77oo%2BDOXuG2CpwvoKPnNTKOGH91AkCHVdm0t6XKQEEnAFP%2BQ35cF49Y%2BAviwQKVN04%2B6ZbeKc2tNOeeC5vfTZ4Cme%2BwZVk7zGkwHJbfjgp1J9Y30o1fMKHOE5rxyhwQw%2B%5CDH6Md%5CpJZAAh2xkZ%3A1536204296617; __utma=94650624.1052021654.1536048185.1536199016.1536203113.27; __utmb=94650624.12.10.1536203113',
'Host': 'music.163.com',
'Referer': 'http://music.163.com/',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/66.0.3359.181 Safari/537.36'}
try:
r = requests.post(url, headers=headers, data=data)
r.encoding = "utf-8"
if r.status_code == 200: # 状态码 200代表服务器正常响应
# 返回json格式的数据
return r.json()
except:
print("爬取失败!")
评论分为两种，一种是精彩评论即我们所说的热评，另一种就是最新评论，按时间排序的maybe
接下来只需要解析网页内容，两种评论只是所在的路径不同，就可以获取到评论了
在获取评论后，利用tkinter写个GUI界面，用于显示爬取的内容，同时将爬取的评论也写入文件中存储
import math
import tkinter
from tkinter import *
from Netease_cloud_comment_capture.searchMusic import search
# 搜索歌曲名称
from Netease_cloud_comment_capture.tool import get_params, get_comments_json, get_encSecKey
def get_music_name():
d = search()
song_id = d.search_song(entry.get())
text.insert(END, '解析到歌曲的id为：{}\n'.format(song_id))
text.update()
# 歌曲名字
songname = entry.get()
# 文件存储路径
filepath = songname + ".txt"
page = 1
params = get_params(1)
encSecKey = get_encSecKey()
url = 'https://music.163.com/weapi/v1/resource/comments/R_SO_4_' + str(song_id) + '?csrf_token='
data = {'params': params, 'encSecKey': encSecKey}
# 获取第一页评论
html = get_comments_json(url, data)
# 评论总数
total = html['total']
# 总页数
pages = math.ceil(total / 20)
if(total>5):
pages = 5
else:
pages = total
hotcomments(html, songname, page, pages, total, filepath)
comments(html, songname, page, pages, total, filepath)
# 开始获取歌曲的全部评论
page = 2
while page <= pages:
params = get_params(page)
encSecKey = get_encSecKey()
data = {'params': params, 'encSecKey': encSecKey}
html = get_comments_json(url, data)
# 从第二页开始获取评论
comments(html, songname, page, pages, total, filepath)
page += 1
tkinter.messagebox.showinfo('提示', '评论抓取完成，请查看！')
########
def hotcomments(html, songname, i, pages, total, filepath):
text.insert(END, '加载中，请稍等！\n')
text.update()
text.after(100)
# 写入文件
with open(filepath, 'a', encoding='utf-8') as f:
f.write("正在获取歌曲{}的第{}页评论, 总共有{}页{}条评论！\n\n".format(songname, i, pages, total))
text.insert(END, "正在获取歌曲{}的第{}页评论, 总共有{}页{}条评论！\n\n".format(songname, i, pages, total))
text.update()
text.after(100)
# 精彩评论
m = 1
# 键在字典中则返回True, 否则返回False
if 'hotComments' in html:
for item in html['hotComments']:
# 提取发表热门评论的用户名
user = item['user']
# 写入文件
text.insert(END, " 热门评论{} 用户名：{} 点赞次数: {}\n\n".format(m, user['nickname'],item['likedCount']))
text.insert(END, " 评论内容：{}\n\n".format(item['content']))
text.update()
text.after(100)
with open(filepath, 'a', encoding='utf-8') as f:
f.write(" 热门评论{} 用户名：{} 点赞次数: {}\n\n".format(m, user['nickname'], item['likedCount']))
f.write(" 评论内容：{}\n\n".format(item['content']))
text.insert(END, "\n\n")
# 回复评论
if len(item['beReplied']) != 0:
for reply in item['beReplied']:
# 提取发表回复评论的用户名
replyuser = reply['user']
text.insert(END, " 回复：{} : {}".format(replyuser['nickname'], reply['content']))
text.insert(END, "\n\n")
text.update()
text.after(100)
f.write(" 回复：{} : {}\n".format(replyuser['nickname'], reply['content']))
m += 1
def comments(html, songname, i, pages, total, filepath):
with open(filepath, 'a', encoding='utf-8') as f:
f.write("\n\n正在获取歌曲{}的第{}页评论, 总共有{}页{}条评论！\n".format(songname, i, pages, total))
text.insert(END, "\n\n正在获取歌曲{}的第{}页评论, 总共有{}页{}条评论！\n".format(songname, i, pages, total))
text.update()
text.after(100)
# 全部评论
j = 1
for item in html['comments']:
# 提取发表评论的用户名
user = item['user']
text.insert(END, " 最新评论{} 用户名：{} 点赞次数: {}\n\n".format(j, user['nickname'],item['likedCount']))
text.insert(END, " 评论内容：{}\n\n".format(item['content']))
text.insert(END, "\n\n")
text.update()
text.after(10)
with open(filepath, 'a', encoding='utf-8') as f:
f.write(" 最新评论{} 用户名：{} 点赞次数: {}\n\n".format(j, user['nickname'], item['likedCount']))
f.write(" 评论内容：{}\n\n".format(item['content']))
text.insert(END, "\n\n")
# 回复评论
if len(item['beReplied']) != 0:
for reply in item['beReplied']:
# 提取发表回复评论的用户名
replyuser = reply['user']
text.insert(END, " 回复：{} : {}".format(replyuser['nickname'], reply['content']))
text.insert(END, "\n\n")
text.update()
text.after(10)
f.write(" 回复：{} : {}\n".format(replyuser['nickname'], reply['content']))
j += 1
# 创建界面
root = Tk()
# 标题
root.title("网易云评论爬取脚本")
# 设置窗口大小
root.geometry('1123x410')
root.configure(bg="#FFFFDF")
# 标签控件
label = Label(root, text='请输入要爬取的歌曲名称：', font=('幼圆', 15,), bg='#FAF4FF')
# 标签定位
label.grid(sticky=W)
# 输入框
entry = Entry(root, font=('幼圆', 15), bg='#ECECFF')
entry.grid(row=0, column=1, sticky=W)
# 抓取按钮
button = Button(root, text='抓取评论', font=('幼圆', 15), command=get_music_name, bg='#CEFFCE')
# 左对齐
button.grid(row=0, column=2, sticky=W)
# 列表框
text = Listbox(root, font=('幼圆', 16), width=100, height=20, bg='#E6E6F2')
text.grid(row=1, columnspan=4)
# 退出按钮
button1 = Button(root, text='退出', font=('幼圆', 15), command=root.quit, bg='#CAFFFF')
button1.grid(row=0, column=3, sticky=E)
# 显示界面
root.mainloop()
