一、完整代码
bk.py
import json
import scrapy
from ScrapyAdvanced.items import HouseItem
class BkSpider(scrapy.Spider):
name = 'bk'
allowed_domains = ['bj.ke.com']
start_urls = ['https://bj.ke.com/ershoufang/']
def parse(self, response):
# house_list_position = response.xpath("//div[@class='address']/div/div/a/text()").extract()
# # 存在脏数据 一个空的脏数据 一个 正常数据
# house_list_info = response.xpath("//div[@class='address']/div[@class='houseInfo']/text()").extract()
house_list_address = response.xpath("//div[@class='address']")
house_list_titles = response.xpath("//div[@class='title']/a/text()").extract()
for i in range(len(house_list_address)):
title = house_list_titles[i]
house = house_list_address[i]
position = house.xpath("./div/div/a/text()").extract_first()
info = "".join(house.xpath("./div[@class='houseInfo']/text()").extract()).replace("\n", "").replace(" ","")
tags = ";".join(house.xpath("./div[@class='tag']/span/text()").extract())
total_price = house.xpath("./div[@class='priceInfo']/div[@class='totalPrice']/span/text()").extract_first() + "万"
unit_price = house.xpath("./div[@class='priceInfo']/div[@class='unitPrice']/span/text()").extract_first()
house_item = HouseItem()
house_item["title"] = title
house_item["position"] = position
house_item["info"] = info
house_item["tags"] = tags
house_item["total_price"] = total_price
house_item["unit_price"] = unit_price
yield house_item
page_info = json.loads(response.xpath("//div[@class='page-box house-lst-page-box']/@page-data").extract_first())
page_url = response.xpath("//div[@class='page-box house-lst-page-box']/@page-url").extract_first()
total_page = page_info.get("totalPage")
current_page = page_info.get("curPage")
# 当前页面不是总页面的时候, 也就是说 不是最后一页, 去请求下一页
if current_page != total_page:
url = page_url.replace("{page}", str(current_page+1) + "/")
yield response.follow(url=url, callback=self.parse)
# 当前页是第一页,发送请求后面所有的页
# if current_page == 1:
# for i in range(2, total_page):
# url = page_url.replace("{page}", str(i))
# yield response.follow(url=url, callback=self.parse)
items.py
class HouseItem(scrapy.Item):
title = scrapy.Field()
position = scrapy.Field()
info = scrapy.Field()
tags = scrapy.Field()
total_price = scrapy.Field()
unit_price = scrapy.Field()
pipelines.py
class HousePipeline():
def process_item(self, item, spider):
if isinstance(item, HouseItem) and isinstance(spider, BkSpider):
# 存储
with open("house_list.csv", "a", encoding="utf-8") as house_file:
csv_writer = csv.writer(house_file)
row = [it[1] for it in item.items()]
csv_writer.writerow(row)
return item