我有一个链接，在该链接中我有一些产品。每个产品中都有一个规格表。该表的第一列应为标题，第二列应为与其对应的数据。每个表的第一列都不同，有一些重叠的类别。我想要一张大表，其中包含所有这些类别，并且按行排列不同的产品。我能够获取一张表（一种产品）的数据，如下所示：
import requests
import csv
from bs4 import BeautifulSoup
def cpap_spider(max_pages):
page=1
while page<=max_pages:
url= "https://www.1800cpap.com/cpap-masks/nasal?page=" +str(page)
source_code= requests.get(url)
plain_text= source_code.text
soup= BeautifulSoup(plain_text, 'html.parser')
for link in soup.findAll("a", {"class":"facets-item-cell-grid-title"}):
href="https://www.1800cpap.com"+link.get("href")
title= link.string
each_item(href)
print(href)
#print(title)
page+=1
data=[]
def each_item(item_url):
source_code= requests.get(item_url)
plain_text= source_code.text
soup= BeautifulSoup(plain_text, 'html.parser')
table=soup.find("table", {"class":"table"})
table_rows= table.find_all('tr')
for row in table_rows:
cols = row.find_all('td')
cols = [ele.text.strip() for ele in cols]
data.append([ele for ele in cols if ele]) # Get rid of empty values
b = open('all_appended.csv', 'w')
a = csv.writer(b)
a.writerows(data)
b.close()
cpap_spider(1)
此代码将所有表一个接一个地附加。但是，我想要一个大表，其第一行具有唯一的标题，并且按顺序排列相应的产品值。
Use xlsxwriter代替csv因为如果文本包含一个逗号，旁边没有空格","而不是逗号旁边有空格", "那么您的 csv 文件将导致问题，因为每个列值都由","例如如果text = "aa,bb"那么 csv 会认为这个文本包含两列，例如"aa" and "bb".
这就是你需要的
import requests
import xlsxwriter
from bs4 import BeautifulSoup
def cpap_spider(max_pages):
global row_i
page=1
while page<=max_pages:
url= "https://www.1800cpap.com/cpap-masks/nasal?page=" +str(page)
source_code= requests.get(url)
plain_text= source_code.text
soup= BeautifulSoup(plain_text, 'html.parser')
for link in soup.findAll("a", {"class":"facets-item-cell-grid-title"}):
href="https://www.1800cpap.com"+link.get("href")
title = link.string
worksheet.write(row_i, 0, title)
each_item(href)
print(href)
#print(title)
page+=1
def each_item(item_url):
global cols_names, row_i
source_code= requests.get(item_url)
plain_text= source_code.text
soup= BeautifulSoup(plain_text, 'html.parser')
table=soup.find("table", {"class":"table"})
if table:
table_rows = table.find_all('tr')
else:
return
for row in table_rows:
cols = row.find_all('td')
for ele in range(0,len(cols)):
temp = cols[ele].text.strip()
if temp:
# Here if you want then you can remove unwanted characters like : ? from temp
# For example "Actual Weight" and ""
if temp[-1:] == ":":
temp = temp[:-1]
# Name of column
if ele == 0:
try:
cols_names_i = cols_names.index(temp)
except:
cols_names.append(temp)
cols_names_i = len(cols_names) - 1
worksheet.write(0, cols_names_i + 1, temp)
continue;
worksheet.write(row_i, cols_names_i + 1, temp)
row_i += 1
cols_names=[]
cols_names_i = 0
row_i = 1
workbook = xlsxwriter.Workbook('all_appended.xlsx')
worksheet = workbook.add_worksheet()
worksheet.write(0, 0, "Title")
cpap_spider(1)
#each_item("https://www.1800cpap.com/viva-nasal-cpap-mask-by-3b-medical")
workbook.close()
本文内容由网友自发贡献，版权归原作者所有，本站不承担相应法律责任。如您发现有涉嫌抄袭侵权的内容，请联系:hwhale#tublm.com(使用前将#替换为@)
