自己本人在暑期时自学了python，还在中国大学mooc上学习了一些爬虫相关的知识，对requests库、re库以及BeautifulSoup库有了一定的了解，但是没有过爬虫方面的实战，刚好家人有这方面需求，就对58同城上的租房数据进行了爬取，算是当作自己的第一次爬虫实战。不过自己的水平实在不行，搞了两天，参考了很多人的代码，也查了很多文件。好，话不多说，直接上代码。
from bs4 import BeautifulSoup
from fontTools.ttLib import TTFont
import requests
import json
import time
import re
import base64
import openpyxl
User_Agent = 'Mozilla/5.0(Macintosh;IntelMacOSX10_7_0)AppleWebKit/535.11(KHTML,likeGecko)Chrome/17.0.963.56Safari/535.11'
headers = {
'User-Agent': User_Agent,
}
def convertNumber(html_page):
base_fonts = ['uni9FA4', 'uni9F92', 'uni9A4B', 'uni9EA3', 'uni993C', 'uni958F', 'uni9FA5', 'uni9476', 'uni9F64',
'uni9E3A']
base_fonts2 = ['&#x' + x[3:].lower() + ';' for x in base_fonts] # 构造成 鸺 的形式
pattern = '(' + '|'.join(base_fonts2) + ')'
font_base64 = re.findall("base64,(AA.*AAAA)", html_page)[0] # 找到base64编码的字体格式文件
font = base64.b64decode(font_base64)
with open('58font2.ttf', 'wb') as tf:
tf.write(font)
onlinefont = TTFont('58font2.ttf')
convert_dict = onlinefont['cmap'].tables[0].ttFont.tables['cmap'].tables[0].cmap # convert_dict数据如下：{40611: 'glyph00004', 40804: 'glyph00009', 40869: 'glyph00010', 39499: 'glyph00003'
new_page = re.sub(pattern, lambda x: getNumber(x.group(),convert_dict), html_page)
return new_page
def getNumber(g,convert_dict):
key = int(g[3:7], 16) # '麣',截取后四位十六进制数字，转换为十进制数，即为上面字典convert_dict中的键
number = int(convert_dict[key][-2:]) - 1 # glyph00009代表数字8， glyph00008代表数字7，依次类推
return str(number)
def request_details(url):
f = requests.get(url, headers=headers)
f=convertNumber(f.text)
soup = BeautifulSoup(f, 'lxml')
name = soup.select('h1[class="c_333 f20 strongbox"]')
price = soup.select('b[class="f36 strongbox"]')
try:
lease=soup.find(text='租赁方式：').findNext('span').contents[0]
except:
lease=""
try:
housing_type=soup.find(text='房屋类型：').findNext('span').contents[0]
housing_type="".join(housing_type.split())
except:
housing_type=""
try:
oritation=soup.find(text='朝向楼层：').findNext('span').contents[0]
oritation="".join(oritation.split())
except:
oritation=""
try:
community=soup.find(text='所在小区：').findNext('span').contents[0].contents[0]
except:
community=""
try:
building_type=soup.find(text='建筑类型：').findNext('span').contents[0]
except:
building_type=""
try:
address=soup.find(text='详细地址：').findNext('span').contents[0]
address="".join(address.split())
except:
address=""
data = {
'名称':name[0].text,
'价格':price[0].text.strip()+"元",
'租赁方式':lease,
'房屋类型':housing_type,
'朝向楼层':oritation,
'所在小区':community,
'建筑类型':building_type,
'详细地址':address,
'网址':url
}
return data
def get_link(url):
f = requests.get(url)
soup = BeautifulSoup(f.text, 'lxml')
links = soup.select('div[class="des"]>h2>a')
link_list = []
for link in links:
link_content = link.get('href')
link_list.append(link_content)
return link_list
def save_to_text(content,i):
mywb=openpyxl.load_workbook('SHEET2.xlsx')
sheet=mywb.active;
sheet[chr(65)+str(i)].value=content['名称']
sheet[chr(66)+str(i)].value=content['价格']
sheet[chr(67)+str(i)].value=content['租赁方式'].encode('utf-8')
sheet[chr(68)+str(i)].value=content['房屋类型'].encode('utf-8')
sheet[chr(69)+str(i)].value=content['朝向楼层'].encode('utf-8')
sheet[chr(70)+str(i)].value=content['所在小区'].encode('utf-8')
sheet[chr(71)+str(i)].value=content['建筑类型'].encode('utf-8')
sheet[chr(72)+str(i)].value=content['详细地址'].encode('utf-8')
sheet[chr(73)+str(i)].value=content['网址']
mywb.save('SHEET2.xlsx')
content = json.dumps(content, ensure_ascii=False)
with open('58', 'a', encoding='utf-8') as f:
f.write(content)
f.write('\r\n')
def main():
mywb=openpyxl.Workbook()
sheet=mywb.active;
sheet[chr(65)+str(1)].value="名称"
sheet[chr(66)+str(1)].value="价格"
sheet[chr(67)+str(1)].value="租赁方式"
sheet[chr(68)+str(1)].value="房屋类型"
sheet[chr(69)+str(1)].value="朝向楼层"
sheet[chr(70)+str(1)].value="所在小区"
sheet[chr(71)+str(1)].value="建筑类型"
sheet[chr(72)+str(1)].value="详细地址"
sheet[chr(73)+str(1)].value="网址"
mywb.save('SHEET2.xlsx')
num = 1
link = 'https://fy.58.com/qhdlfyzq/chuzu/pn{}'
start = 1
end = 11
urls = [link.format(i) for i in range(start, end)]
for url in urls:
link_list = get_link(url)
print(link_list)
time.sleep(5)
for link in link_list:
if link != "https://e.58.com/all/zhiding.html":
try:
content = request_details(link)
num = num + 1
print(content)
save_to_text(content, num)
except:
print("error")
time.sleep(5)
if __name__ == '__main__':
main()
