Html文件读取与爬取（个人笔记篇）
文章目录
- Html文件读取与爬取（个人笔记篇）
- html文件读取
- 解析html数据
- 存储爬取到的数据
- 完整代码展示
html文件读取
1.通过open：打开文件地址，‘r’：读取文件，encoding：读取文件的格式
2.再通过html.read()
3.获取到html
def get_html(self, url):
html = open(url, 'r', encoding='utf-8')
h = html.read()
return h
解析html数据
1、利用bs4 进行解析数据
2、在通过循环利用item｛｝存储数据
再将保存在item中的数据存储到data_list中
def parse_data(self, html):
soup = BeautifulSoup(html, 'lxml')
tr_list = soup.find_all('tr')
for index, tr in enumerate(tr_list):
if index == 0:
continue
else:
td_list = tr.find_all('td')
item = {}
td1 = td_list[0]
item['courseId'] = td1.string
td2 = td_list[1]
item['name'] = td2.string
td3 = td_list[2]
item['university'] = td3.string
td4 = td_list[3]
item['summary'] = td4.string
td5 = td_list[4]
item['teacher'] = td5.string
td6 = td_list[5]
item['term'] = td6.string
td7 = td_list[6]
item['assessment_number'] = td7.string
td8 = td_list[7]
item['assessment_score'] = td8.string
td9 = td_list[8]
item['category'] = td9.string
self.data_list.append(item)
存储爬取到的数据
利用with open以及csv模块创建和存储爬取到的数据
def __init__(self):
self.header = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/97.0.4692.99 Safari/537.36 "
}
self.head = ['courseId', 'name', 'university', 'summary', 'teacher', 'term', 'assessment_number',
'assessment_score', 'category']
self.data_list = []
def save_data(self):
with open('train.csv', 'w', encoding='utf-8', newline='') as file_obj:
DictW = csv.DictWriter(file_obj, self.head)
DictW.writeheader()
DictW.writerows(self.data_list)
完整代码展示
import csv
from bs4 import BeautifulSoup
class Html_pq():
def __init__(self):
self.header = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/97.0.4692.99 Safari/537.36 "
}
self.head = ['courseId', 'name', 'university', 'summary', 'teacher', 'term', 'assessment_number',
'assessment_score', 'category']
self.data_list = []
def get_html(self, url):
html = open(url, 'r', encoding='utf-8')
h = html.read()
return h
def parse_data(self, html):
soup = BeautifulSoup(html, 'lxml')
tr_list = soup.find_all('tr')
for index, tr in enumerate(tr_list):
if index == 0:
continue
else:
td_list = tr.find_all('td')
item = {}
td1 = td_list[0]
item['courseId'] = td1.string
td2 = td_list[1]
item['name'] = td2.string
td3 = td_list[2]
item['university'] = td3.string
td4 = td_list[3]
item['summary'] = td4.string
td5 = td_list[4]
item['teacher'] = td5.string
td6 = td_list[5]
item['term'] = td6.string
td7 = td_list[6]
item['assessment_number'] = td7.string
td8 = td_list[7]
item['assessment_score'] = td8.string
td9 = td_list[8]
item['category'] = td9.string
self.data_list.append(item)
def save_data(self):
with open('train.csv', 'w', encoding='utf-8', newline='') as file_obj:
DictW = csv.DictWriter(file_obj, self.head)
DictW.writeheader()
DictW.writerows(self.data_list)
def main(self):
url = './2.Train.html'
html = self.get_html(url)
self.parse_data(html)
self.save_data()
if __name__ == '__main__':
pq = Html_pq()
pq.main()
本文内容由网友自发贡献，版权归原作者所有，本站不承担相应法律责任。如您发现有涉嫌抄袭侵权的内容，请联系:hwhale#tublm.com(使用前将#替换为@)
