使用python抓取天涯帖子，并实现“只看楼主”功能
在看天涯论坛时，有些帖子很有价值，想把它保存下来，可是好的帖子动辄几百页，其中又掺杂了很多灌水和撕逼，所以我编写了一个python小程序，将一个帖子中楼主所发的内容抓取下来，挺好用的。
唯一的缺陷是图片还没法抓取，以后有空再改进。
下面是程序代码，注释都在里面了：
#!/usr/bin/python
# -*- coding: utf-8 -*-
# 天涯帖子下载，“只看楼主”
from bs4 import BeautifulSoup
import codecs
import re
import requests
import time
# 主下载函数
def download_post(home_page, link_prefix):
# 打开文件
filename = 'D:/temp/post.txt'
f = codecs.open(filename, 'w', 'utf-8')
########## 数据抓取 ##########
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.82 Safari/537.36'}
home_page_file = requests.get(home_page, headers=headers)
home_page_file.encoding = 'utf-8'
home_page_file = home_page_file.text
soup = BeautifulSoup(home_page_file, 'html.parser')
########## 获取帖子总体信息，例如标题、作者、时间等 ##########
# 帖子标题
post_title = soup.select('h1.atl-title > span > span')[0].string.strip()
post_info = soup.select('div.atl-info > span')
# 帖子作者
post_author = post_info[0].select('a')[0].string.strip()
# 帖子时间
post_time = post_info[1].string[3:].strip()
f.write(u'帖子标题：' + post_title + '\r\n')
f.write(u'帖子作者：' + post_author + '\r\n')
f.write(u'帖子时间：' + post_time + '\r\n\r\n')
f.write(u'#################### 我是分隔线 ####################\r\n\r\n')
########## 获取帖子的内容 ##########
post_items = soup.select('div.atl-item')
for i in range(len(post_items)):
# 主帖，主帖和跟帖的区别是跟帖有js_username属性
if not post_items[i].get('js_username'):
post_content = unicode(post_items[i].select('div.bbs-content')[0])
post_content = process_post_content(post_content)
if post_content:
f.write(u' ' + post_content + '\r\n\r\n')
f.write(u'#################### 我是分隔线 ####################\r\n\r\n')
# 跟帖
else:
# 只看楼主
if post_items[i].get('js_username') == post_author:
post_time = post_items[i].select(
'div.atl-info > span')[1].string.strip()
f.write(u' ' + post_time + '\r\n\r\n')
post_content = unicode(
post_items[i].select('div.bbs-content')[0])
post_content = process_post_content(post_content)
if post_content:
f.write(u' ' + post_content + '\r\n\r\n')
f.write(
u'#################### 我是分隔线 ####################\r\n\r\n')
########## 处理翻页 ##########
# 获取翻页链接
next_page = soup.select('div.atl-pages > form > a.js-keyboard-next')
while next_page:
next_page_href = next_page[0].get('href')
########## 数据抓取 ##########
home_page_file = requests.get(
link_prefix + next_page_href, headers=headers)
home_page_file.encoding = 'utf-8'
home_page_file = home_page_file.text
soup = BeautifulSoup(home_page_file, 'html.parser')
########## 获取帖子的内容 ##########
post_items = soup.select('div.atl-item')
for i in range(len(post_items)):
# 只看楼主
if post_items[i].get('js_username') == post_author:
post_time = post_items[i].select(
'div.atl-info > span')[1].string.strip()
f.write(u' ' + post_time + '\r\n\r\n')
post_content = unicode(
post_items[i].select('div.bbs-content')[0])
post_content = process_post_content(post_content)
if post_content:
f.write(u' ' + post_content + '\r\n\r\n')
f.write(
u'#################### 我是分隔线 ####################\r\n\r\n')
# 获取翻页链接
next_page = soup.select('div.atl-pages > form > a.js-keyboard-next')
# 关闭文件
f.close()
# 帖子内容处理函数
def process_post_content(post_content):
post_content = post_content.replace(
'<div class="bbs-content clearfix">', '')
post_content = post_content.replace('<div class="bbs-content">', '')
post_content = post_content.replace('</div>', '')
post_content = post_content.replace('<br/>', '\r\n')
post_content = post_content.strip()
return post_content
# 天涯帖子首页，每次抓取帖子前修改这个值
home_page = 'http://bbs.tianya.cn/post-worldlook-703166-1.shtml'
# 翻页时，各页面链接需要加上的前缀
link_prefix = 'http://bbs.tianya.cn'
download_post(home_page, link_prefix)
