import requests import json import re import time import simplejson import xlsxwriter workbook = xlsxwriter.Workbook('kebiao_test.xlsx') worksheet = workbook.add_worksheet() worksheet.set_column('A:A', 10) worksheet.set_column('B:B', 20) worksheet.set_column('C:C', 5) worksheet.set_column('D:D', 20) worksheet.set_column('E:E', 20) worksheet.set_column('F:F', 500) worksheet.write(0,0,'学校') worksheet.write(0,1,'主题') worksheet.write(0,2,'性别') worksheet.write(0,3,'日期') worksheet.write(0,4,'时间') worksheet.write(0,5,'内容') # 登陆部分 # 用requests.Session()记录cookie s = requests.Session() loginurl = 'http://120.55.151.61/V2/StudentSkip/loginCheckV4.action' logindata = 'phoneBrand=Xiaomi&platform=1&deviceCode=867711022104024 &account=6603135B883F9B40DED6374A22593&phoneVersion=19 &password=98A09B5680DF25A934ACF9B3614AF4EA&channel=XiaoMiMarket &phoneModel=HM+2A&versionNumber=7.4.1&' headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'User-Agent': 'Dalvik/1.6.0 (Linux; U; Android 4.4.4; HM 2A MIUI/V7.3.1.0.KHLCNDD)', 'Host': '120.55.151.61', 'Connection': 'Keep-Alive', 'Accept-Encoding': 'gzip', 'Content-Length': '213', } # 提交登陆信息,开始登陆 data = s.post(url=loginurl, data=logindata, headers=headers, stream=True, verify=False) loginResult = data.text # 打印登陆信息,检查是否登陆成功 # 成功后返回个人信息 print(loginResult) # 定义一个函数,获取信息并保存 # 传入要POST的表单数据,即req_data def get_data(req_data): req_url = 'http://120.55.151.61/Treehole/V4/Cave/getList.action' data_r = s.post(url=req_url, data=req_data, headers=headers) data_r = data_r.text # 返回的布尔值为小写,这里转化为真正的布尔值 true = True false = False # 简单处理json数据,直接解析好像有编码问题,不太清楚。。。 data_j = eval(data_r) data_js = json.dumps(data_j) data_dict = simplejson.loads(data_js) # 获取时间戳 data = data_dict['data'] timestampLong = data['timestampLong'] #print(timestampLong) messageBO = data['messageBOs'] # 处理抓到的json数据,拿到目标数据并保存 for each in messageBO: if 'studentBO' in each: print(each) #topicDict = {} if each.get('content', False): schoolNamex = each["schoolName"] worksheet.write(i,0,schoolNamex) tag = each['moodTagBO']['moodTagName'] worksheet.write(i,1,tag) genderx = each['studentBO']['gender'] worksheet.write(i,2,genderx) contentx = each['content'] #print(contentx) worksheet.write(i,5,contentx) #topicDict['messageId'] = each['messageId' time_f = list(time.localtime(int(str(each['issueTime'])[:-3]))) time_s = str(time_f[0])+'/'+str(time_f[1])+'/'+str(time_f[2]) h = time_f[3] m = time_f[4] sec = time_f[5] if h < 10: h = '0'+str(h) if m < 10: m = '0'+str(m) if sec < 10: sec = '0'+str(sec) time_t = str(h)+':'+str(m)+':'+str(sec) datex = time_s # print(datex) worksheet.write(i,3,datex) worksheet.write(i,4,time_t) i += 1 global i # 用获取的时间戳构建新的POST表单数据,持续循环抓取 new_req_data = 'timestamp='+str(timestampLong)+ '&preMoodTimestap=1461641185406&phoneBrand=Xiaomi &platform=1&phoneVersion=19&channel=XiaoMiMarket &phoneModel=HM+2A&versionNumber=7.4.1&' print('--------------------------new page---------') #print(new_req_data) # 定义抓取发帖条数 if i <= 20000: try: get_data(new_req_data) except: print('fail') finally: workbook.close() else: workbook.close() print('全部抓取成功了!!!') # i 是抓取条数的标识 i = 1 # 此处传入第一个页面的POST数据,之后展开循环 get_data(req_data='timestamp=146646454916&preMoodTimestap=1461641185406 &phoneBrand=Xiaomi&platform=1&phoneVersion=19&channel=XiaoMiMarket &phoneModel=HM+2A&versionNumber=7.4.1&') |