数据分析 告诉你《飞驰人生》为什么这么燃?

数据分析《飞驰人生》为什么这么燃?

2019年贺岁电影《飞驰人生》在豆瓣排名和猫眼排名中都排名第二的位置
我们这里爬取了豆瓣500条的评论数据来，分析一下《飞驰人生》为何这么燃

（这里说明一下，之所以获取这么点数据，是豆瓣的 的反爬限制，非登录状态只能爬取200条，登录状态可以爬取500条)

运行环境:python3.6
需要安装的包:
requests
lmxl
pandas
numpy
matplotlib
jieba
wordcloud

2 数据采集

直接上代码

"""
豆瓣爬取 电影 <<飞驰人生>> 影评和评分
上映日期: 2019-02-05(中国大陆)
人名:name
时间:data_time
评分:score
简评:comment_short
评论投票:votes
"""
import requests,time,json,random
from lxml import etree
import re

def fly(headers,url):
	flying = []
	name_list=[]
	score_list=[]
	votes_list=[]
	comment_short_list=[]
	data_time_list=[]
	for i in range(0,480,20):
		new_url = url.format(i)
		requ = requests.get(new_url,headers=headers)
		req = requ.text
		content = etree.HTML(req)
		name = content.xpath("//div[@class='comment-item']//span[@class='comment-info']/a//text()")
		data_time = re.findall('<span class="comment-time.*title="(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*">',req)
		votes = content.xpath("//div[@id='comments']/div/div[2]/h3/span[1]/span[@class]//text()")
		comment_short = content.xpath("//div[@id='comments']/div/div[2]/p[@class]/span[@class]//text()")

		#score里面有一个bug，如果有的人没有评分，但是评论了，那么score解析出来是日期，而日期所在位置spen[3]为空
		fscore = content.xpath("//div[@id='comments']/div/div[2]/h3/span[2]/span[2]/@title")
		# data = content.xpath("//div[@id='comments']/div/div[2]/h3/span[2]/span[3]/@title")
		# 定义一个正则
		s = '\d{4}-\d{2}-\d{2}'
		score =[]
		# 将匹配的fscore转化为分数
		for i in fscore:
			mt= re.compile(s).match(i)
			if mt is not None:
				i = 0
			elif i =='很差':
				i = 2
			elif i == '较差':
				i = 4
			elif i == '还行':
				i = 6
			elif i == '推荐':
				i = 8
			else:
				i = 10
			score.append(i)
		# 防止ip被封
		time.sleep(round(random.uniform(7, 14), 2))
		name_list.extend(name)
		score_list.extend(score)
		votes_list.extend(votes)
		comment_short_list.extend(comment_short)
		data_time_list.extend(data_time)
	flying.append({
   
		'name':name_list,
		'score':score_list,
		'votes':votes_list,
		'comment_short':comment_short_list,
		'data_time':data_time_list
	})

	fl = json.dumps(flying,ensure_ascii=False)
# 	写入文件
	print(len(flying))
	with open('flying.txt','a+',encoding='utf-8') as f:
		f.write(fl)
if __name__ == '__main__':
	headers = {
   
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
		'Connection': 'keep - alive',
		'cookie':'...'
	}
	url ='https://movie.douban.com/subject/30163509/comments?start={}&limit=20&sort=new_score&status=P'
	fly(headers,url)

3 数据处理

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import json

with open('flying.txt','r',encoding='utf-8') as f:
    fly = f.read()
flying = json.loads(fly)
flying

f = pd.DataFrame(
    flying[0]
)

f.tail()

f.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 480 entries, 0 to 479
Data columns (total 5 columns):
name             480 non-null object
score            480 non-null int64
votes            480 non-null object
comment_short    480 non-null object
data_time        480 non-null object
dtypes: int64(1), object(4)
memory usage: 18.8+ KB

# 将data_time转为时间序列
f['data_time'] = pd.to_datetime(f['data_time'])
f.head()

f.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 480 entries, 0 to 479
Data columns (total 5 columns):
name             480 non-null object
score            480 non-null int64
votes            480 non-null object
comment_short    480 non-null object
data_time        480 non-null datetime64[ns]
dtypes: datetime64[ns](1), int64(1), object(3)
memory usage: 18.8+ KB

# 去除缺失值得行
f.dropna(axis='rows')
f.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 480 entries, 0 to 479
Data columns (total 5 columns):
name             480 non-null object
score            480 non-null int64
votes            480 non-null object
comment_short    480 non-null object
data_time        480 non-null datetime64[ns]
dtypes: datetime64[ns](1), int64(1), object(3)
memory usage: 18.8+ KB

# 将votes转换为整形
f['votes']=f['votes'].astype(np.int)

f.tail()

f.info()

<class 'pandas.core.frame.DataFrame&