如何修复 AttributeError: 'NoneType' 对象没有属性 'text'...循环时

我是初学者，这个论坛上的答案非常宝贵。我正在使用 Python 3 和 Beautiful Soup 通过循环页码从同一网站上的多个网页中抓取（非表）数据。它有效，但我不断收到 AttributeError: 'NoneType' object has no attribute 'text' 第一次迭代后。

这是我迄今为止尝试过的代码：

import requests

from bs4 import BeautifulSoup

import csv

import lxml


# Lists to store the scraped data in

addresses = []
geographies = []
rents = []
units = []
availabilities = []

# Scraping all pages

pages_url = requests.get('https://www.rent.com/new-york/tuckahoe-apartments')

pages_soup = BeautifulSoup(pages_url.text, 'html.parser')

list_nums = pages_soup.find('div', class_='_1y05u').text

print(list_nums)

pages = [str(i) for i in range(1,8)]

for page in pages:

    response = requests.get('https://www.rent.com/new-york/tuckahoe-apartments?page=' + page).text

    html_soup = BeautifulSoup(response, 'lxml')


    # Extract data from individual listing containers

    listing_containers = html_soup.find_all('div', class_='_3PdAH')
    print(type(listing_containers))
    print(len(listing_containers))



    for container in listing_containers:
        address = container.a.text
        addresses.append(address)

        geography = container.find('div', class_='_1dhrl').text
        geographies.append(geography)

        rent = container.find('div', class_='_3e12V').text
        rents.append(rent)

        unit = container.find('div', class_='_2tApa').text
        units.append(unit)

        availability = container.find('div', class_='_2P6xE').text
        availabilities.append(availability)

        import pandas as pd
        test_df = pd.DataFrame({'Street' : addresses,
                                'City-State-Zip' : geographies,
                                'Rent' : rents,
                                'BR/BA' : units,
                                'Units Available' : availabilities

        })
        print(test_df)

这是输出：

240 Properties
<class 'bs4.element.ResultSet'>
30
                     Street                      City-State-Zip     Rent                 BR/BA    Units Available
0  Quarry Place at Tuckahoe  64 Midland PlaceTuckahoe, NY 10707  $2,490+  1–2 Beds • 1–2 Baths  2 Units Available
Traceback (most recent call last):
  File "renttucktabletest.py", line 60, in <module>
    availability = container.find('div', class_='_2P6xE').text
AttributeError: 'NoneType' object has no attribute 'text'

我正在寻找的结果是pandas 数据框中的所有 240 个列表与上面输出中显示的第一次迭代完全相同。任何人都可以帮助修复这个错误吗？将不胜感激。谢谢你！

正如所指出的，问题是一些容器缺少某些div元素。例如，没有“单位”或“可用性”信息。

解决这个问题的一种方法是使用if - else声明。仅当元素存在时追加，否则追加NaN价值。就像是：

import requests
import numpy as np
from bs4 import BeautifulSoup

import csv

import lxml


# Lists to store the scraped data in

addresses = []
geographies = []
rents = []
units = []
availabilities = []

# Scraping all pages

pages_url = requests.get('https://www.rent.com/new-york/tuckahoe-apartments')

pages_soup = BeautifulSoup(pages_url.text, 'html.parser')

list_nums = pages_soup.find('div', class_='_1y05u').text

print(list_nums)

pages = [str(i) for i in range(1,8)]

for page in pages:

    response = requests.get('https://www.rent.com/new-york/tuckahoe-apartments?page=' + page).text

    html_soup = BeautifulSoup(response, 'lxml')


    # Extract data from individual listing containers

    listing_containers = html_soup.find_all('div', class_='_3PdAH')
    print(type(listing_containers))
    print(len(listing_containers))



    for container in listing_containers:
        address = container.a
        if address:
            addresses.append(address.text)
        else:
            addresses.append(np.nan)

        geography = container.find('div', class_='_1dhrl')
        if geography:
            geographies.append(geography.text)
        else:
            geographies.append(np.nan)

        rent = container.find('div', class_='_3e12V')
        if rent:
            rents.append(rent.text)
        else:
            rents.append(np.nan)

        unit = container.find('div', class_='_2tApa')
        if unit:
            units.append(unit.text)
        else:
            units.append(np.nan)

        availability = container.find('div', class_='_2P6xE')
        if availability:
            availabilities.append(availability.text)
        else:
            availabilities.append(np.nan)

import pandas as pd
test_df = pd.DataFrame({'Street' : addresses,
                        'City-State-Zip' : geographies,
                        'Rent' : rents,
                        'BR/BA' : units,
                        'Units Available' : availabilities

})
print(test_df)

                     Street                      City-State-Zip     Rent  \
0  Quarry Place at Tuckahoe  64 Midland PlaceTuckahoe, NY 10707  $2,490+   
1     address not disclosed                  Tuckahoe, NY 10707   $2,510   
2     address not disclosed                  Tuckahoe, NY 10707   $4,145   
3        60 Washington St 1  60 Washington StTuckahoe, NY 10707   $3,500   
4        269 Columbus Ave 5  269 Columbus AveTuckahoe, NY 10707   $2,700   

                  BR/BA    Units Available  
0  1–2 Beds • 1–2 Baths  2 Units Available  
1        1 Bed • 1 Bath                NaN  
2       2 Beds • 2 Bath                NaN  
3       3 Beds • 2 Bath                NaN  
4       2 Beds • 1 Bath                NaN