网页抓取：遍历不同的页面总是从第一页返回内容

Question

Patrick Haugh 是对的，我试图以错误的方式解决这个问题。描述符是要走的路：

from typing import TypeVar, Generic, Optional, ClassVar, Any, Type

FieldValueType = TypeVar('FieldValueType')

class Field(Generic[FieldValueType]):

    value_type: Type[FieldValueType]

    def __init__(self) -> None:

        self.value: FieldValueType = self.value_type()

    def __get__(self, obj, objtype) -> 'Field':

        print('Retrieving', self.__class__)

        return self

    def __set__(self, obj, value):

        print('Updating', self.__class__)

        self.value = value

    def to_string(self):

        return self.value

class StringField(Field[str]):

    value_type = str

class IntField(Field[int]):

    value_type = int

    def to_string(self):

        return str(self.value)

class Person:

    age = IntField()

person = Person()

person.age = 25

print(person.age.to_string())

MyPy可以完全理解这一点。谢谢！

Answer 1

所以使用硒。基本上它获取第一页，然后单击“下一步”。它会一直持续到没有更多的页面可以访问。我遇到的问题是它运行得太快，所以在某些时候它没有找到“Next”并崩溃。我设置了 1 秒的延迟（但有更好的方法可以做到这一点，比如隐式等待……我仍在学习如何正确使用它。）但这会让你继续前进。import bs4 as bsfrom selenium import webdriverimport timeimport pandas as pddriver = webdriver.Chrome('C:/chromedriver_win32/chromedriver.exe')parsed = []url = 'https://etfdb.com/screener/'driver.get(url)while driver.find_element_by_xpath('//*[@id="mobile_table_pills"]/div[1]/div/div[2]/div/ul/li[8]/a'):&nbsp; &nbsp; try:&nbsp; &nbsp; &nbsp; &nbsp; resp = driver.page_source&nbsp; &nbsp; &nbsp; &nbsp; soup = bs.BeautifulSoup(resp, 'lxml')&nbsp; &nbsp; &nbsp; &nbsp; table = soup.find('table', {'class': 'table table-bordered table-hover table-striped mm-mobile-table'})&nbsp; &nbsp; &nbsp; &nbsp; i = 0&nbsp; &nbsp; &nbsp; &nbsp; while i<len(table.find_all('td')):&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; try:&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; ticker = table.find_all('td')[i].text&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; name = table.find_all('td')[i+1].text&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; asset_class = table.find_all('td')[i+2].text&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; parsed.append([ticker, name ,asset_class])&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; except:&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; pass&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; i = i+8&nbsp; &nbsp; &nbsp; &nbsp; elem = driver.find_element_by_xpath('//*[@id="mobile_table_pills"]/div[1]/div/div[2]/div/ul/li[8]/a').click()&nbsp; &nbsp; &nbsp; &nbsp; print ('Aquired page: %s' %(driver.current_url.split('page=')[-1]))&nbsp; &nbsp; &nbsp; &nbsp; time.sleep(1)&nbsp; &nbsp; except:&nbsp; &nbsp; &nbsp; &nbsp; breakdf = pd.DataFrame(parsed, columns=['Ticker','Name','Asset Class'])输出：print (df)&nbsp; &nbsp; &nbsp;Ticker&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; Asset Class0&nbsp; &nbsp; &nbsp; &nbsp;SPY&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity1&nbsp; &nbsp; &nbsp; &nbsp;IVV&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2&nbsp; &nbsp; &nbsp; &nbsp;VTI&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity3&nbsp; &nbsp; &nbsp; &nbsp;VOO&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity4&nbsp; &nbsp; &nbsp; &nbsp;VEA&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity5&nbsp; &nbsp; &nbsp; &nbsp;QQQ&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity6&nbsp; &nbsp; &nbsp; &nbsp;EFA&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity7&nbsp; &nbsp; &nbsp; &nbsp;VWO&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity8&nbsp; &nbsp; &nbsp; IEMG&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity9&nbsp; &nbsp; &nbsp; &nbsp;AGG&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Bond10&nbsp; &nbsp; &nbsp;IEFA&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity11&nbsp; &nbsp; &nbsp; IJH&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity12&nbsp; &nbsp; &nbsp; VTV&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity13&nbsp; &nbsp; &nbsp; IJR&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity14&nbsp; &nbsp; &nbsp; IWM&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity15&nbsp; &nbsp; &nbsp; IWF&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity16&nbsp; &nbsp; &nbsp; IWD&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity17&nbsp; &nbsp; &nbsp; BND&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Bond18&nbsp; &nbsp; &nbsp; VUG&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity19&nbsp; &nbsp; &nbsp; EEM&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity20&nbsp; &nbsp; &nbsp; GLD&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity21&nbsp; &nbsp; &nbsp; VNQ&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; Real Estate22&nbsp; &nbsp; &nbsp; VIG&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity23&nbsp; &nbsp; &nbsp; LQD&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Bond24&nbsp; &nbsp; &nbsp; &nbsp;VB&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity25&nbsp; &nbsp; &nbsp; &nbsp;VO&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity26&nbsp; &nbsp; &nbsp; XLF&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity27&nbsp; &nbsp; &nbsp;VCSH&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Bond28&nbsp; &nbsp; &nbsp;USMV&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity29&nbsp; &nbsp; &nbsp; VEU&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity&nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; ...2219&nbsp; &nbsp; BDD&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity2220&nbsp; &nbsp;WDRW&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2221&nbsp; &nbsp;LACK&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2222&nbsp; &nbsp;HONR&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2223&nbsp; &nbsp;PEXL&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2224&nbsp; FOANC&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2225&nbsp; &nbsp; DYY&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity2226&nbsp; &nbsp;HAUD&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2227&nbsp; &nbsp; SCC&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2228&nbsp; &nbsp;PASS&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2229&nbsp; &nbsp;CHEP&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp;Alternatives2230&nbsp; &nbsp;EKAR&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2231&nbsp; &nbsp; LTL&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2232&nbsp; &nbsp; INR&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Currency2233&nbsp; &nbsp;BUYN&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2234&nbsp; PETZC&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2235&nbsp; &nbsp; SBM&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2236&nbsp; &nbsp;RPUT&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp;Alternatives2237&nbsp; &nbsp; SZO&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity2238&nbsp; &nbsp; EEH&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2239&nbsp; &nbsp;HEWW&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2240&nbsp; &nbsp; FUE&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity2241&nbsp; &nbsp; AGF&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity2242&nbsp; GRBIC&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2243&nbsp; &nbsp; VSL&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Equity2244&nbsp; &nbsp;DLBL&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Bond2245&nbsp; &nbsp; BOS&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity2246&nbsp; &nbsp; &nbsp;LD&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity2247&nbsp; &nbsp; BOM&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity2248&nbsp; &nbsp; DDP&nbsp; &nbsp; &nbsp; ...&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Commodity[2249 rows x 3 columns]

Answer 2

该站点的分页操作是动态的，因此，您需要使用selenium：from selenium import webdriverfrom bs4 import BeautifulSoup as soupimport redef get_content(_d):&nbsp; _headers = [re.sub('\n', '', i.text) for i in _d.find('table').find('tr').find_all('th')]&nbsp; _, *data = list(filter(None, [[re.sub('\n', '', i.text) for i in b.find_all('td')] for b in _d.find_all('tr')]))&nbsp; return [dict(zip(_headers, i)) for i in data]d = webdriver.Chrome('/Users/jamespetullo/Downloads/chromedriver')d.get('https://etfdb.com/screener/')all_data = [get_content(soup(d.page_source, 'html.parser'))]_next = [i for i in d.find_elements_by_tag_name('a') if re.findall('^next', i.text, re.I)]while _next:&nbsp; _next[0].send_keys('\n')&nbsp; all_data.append(get_content(soup(d.page_source, 'html.parser')))&nbsp; _next = [i for i in d.find_elements_by_tag_name('a') if re.findall('^next', i.text, re.I)]示例输出（仅限第一页，由于 SO 的回答字符限制）：[{'Symbol': 'IVV', 'ETF Name': 'iShares Core S&P 500 ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$159,613.22', 'YTD': '11.41%', 'Avg. Volume': '6,062,749', 'Previous Closing Price': '$280.32', 'Overall Rating': ''}, {'Symbol': 'VTI', 'ETF Name': 'Vanguard Total Stock Market ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$107,909.21', 'YTD': '12.40%', 'Avg. Volume': '4,901,940', 'Previous Closing Price': '$143.46', 'Overall Rating': ''}, {'Symbol': 'VOO', 'ETF Name': 'Vanguard S&P 500 ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$102,814.11', 'YTD': '11.43%', 'Avg. Volume': '4,474,837', 'Previous Closing Price': '$256.07', 'Overall Rating': ''}, {'Symbol': 'VEA', 'ETF Name': 'Vanguard FTSE Developed Markets ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$70,063.68', 'YTD': '9.97%', 'Avg. Volume': '22,225,420', 'Previous Closing Price': '$40.80', 'Overall Rating': ''}, {'Symbol': 'QQQ', 'ETF Name': 'Invesco QQQ', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$68,001.87', 'YTD': '12.27%', 'Avg. Volume': '48,660,348', 'Previous Closing Price': '$173.19', 'Overall Rating': ''}, {'Symbol': 'EFA', 'ETF Name': 'iShares MSCI EAFE ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$65,100.01', 'YTD': '9.34%', 'Avg. Volume': '39,227,020', 'Previous Closing Price': '$64.27', 'Overall Rating': ''}, {'Symbol': 'VWO', 'ETF Name': 'Vanguard FTSE Emerging Markets ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$62,464.73', 'YTD': '9.24%', 'Avg. Volume': '21,504,412', 'Previous Closing Price': '$41.62', 'Overall Rating': ''}, {'Symbol': 'IEMG', 'ETF Name': 'iShares Core MSCI Emerging Markets ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$58,650.02', 'YTD': '8.42%', 'Avg. Volume': '23,205,799', 'Previous Closing Price': '$51.12', 'Overall Rating': ''}, {'Symbol': 'AGG', 'ETF Name': 'iShares Core U.S. Aggregate Bond ETF', 'Asset ClassNew': 'Bond', 'Total Assets ($MM)': '$58,023.69', 'YTD': '0.79%', 'Avg. Volume': '6,496,300', 'Previous Closing Price': '$107.07', 'Overall Rating': ''}, {'Symbol': 'IEFA', 'ETF Name': 'iShares Core MSCI EAFE ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$57,206.30', 'YTD': '9.62%', 'Avg. Volume': '15,197,579', 'Previous Closing Price': '$60.29', 'Overall Rating': ''}, {'Symbol': 'IJH', 'ETF Name': 'iShares Core S&P Mid-Cap ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$48,274.19', 'YTD': '15.12%', 'Avg. Volume': '2,089,207', 'Previous Closing Price': '$191.16', 'Overall Rating': ''}, {'Symbol': 'VTV', 'ETF Name': 'Vanguard Value ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$46,173.50', 'YTD': '10.02%', 'Avg. Volume': '2,821,418', 'Previous Closing Price': '$107.76', 'Overall Rating': ''}, {'Symbol': 'IJR', 'ETF Name': 'iShares Core S&P Small-Cap ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$44,864.19', 'YTD': '15.44%', 'Avg. Volume': '5,460,632', 'Previous Closing Price': '$80.02', 'Overall Rating': ''}, {'Symbol': 'IWM', 'ETF Name': 'iShares Russell 2000 ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$43,733.85', 'YTD': '17.09%', 'Avg. Volume': '26,073,227', 'Previous Closing Price': '$156.78', 'Overall Rating': ''}, {'Symbol': 'IWF', 'ETF Name': 'iShares Russell 1000 Growth ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$42,024.32', 'YTD': '12.77%', 'Avg. Volume': '2,928,795', 'Previous Closing Price': '$147.63', 'Overall Rating': ''}, {'Symbol': 'IWD', 'ETF Name': 'iShares Russell 1000 Value ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$38,059.34', 'YTD': '11.08%', 'Avg. Volume': '4,668,100', 'Previous Closing Price': '$123.35', 'Overall Rating': ''}, {'Symbol': 'BND', 'ETF Name': 'Vanguard Total Bond Market ETF', 'Asset ClassNew': 'Bond', 'Total Assets ($MM)': '$37,358.63', 'YTD': '1.02%', 'Avg. Volume': '2,981,882', 'Previous Closing Price': '$79.82', 'Overall Rating': ''}, {'Symbol': 'VUG', 'ETF Name': 'Vanguard Growth ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$36,989.43', 'YTD': '13.23%', 'Avg. Volume': '1,319,368', 'Previous Closing Price': '$152.10', 'Overall Rating': ''}, {'Symbol': 'EEM', 'ETF Name': 'iShares MSCI Emerging Markets ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$34,442.08', 'YTD': '8.65%', 'Avg. Volume': '89,719,367', 'Previous Closing Price': '$42.44', 'Overall Rating': ''}, {'Symbol': 'GLD', 'ETF Name': 'SPDR Gold Trust', 'Asset ClassNew': 'Commodity', 'Total Assets ($MM)': '$33,249.82', 'YTD': '2.26%', 'Avg. Volume': '8,782,053', 'Previous Closing Price': '$123.99', 'Overall Rating': ''}, {'Symbol': 'VNQ', 'ETF Name': 'Vanguard Real Estate Index Fund', 'Asset ClassNew': 'Real Estate', 'Total Assets ($MM)': '$32,604.41', 'YTD': '12.63%', 'Avg. Volume': '8,538,158', 'Previous Closing Price': '$83.99', 'Overall Rating': ''}, {'Symbol': 'VIG', 'ETF Name': 'Vanguard Dividend Appreciation ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$32,584.13', 'YTD': '11.19%', 'Avg. Volume': '1,610,603', 'Previous Closing Price': '$108.91', 'Overall Rating': ''}, {'Symbol': 'LQD', 'ETF Name': 'iShares iBoxx $ Investment Grade Corporate Bond ETF', 'Asset ClassNew': 'Bond', 'Total Assets ($MM)': '$32,520.19', 'YTD': '3.16%', 'Avg. Volume': '9,748,131', 'Previous Closing Price': '$116.03', 'Overall Rating': ''}, {'Symbol': 'VB', 'ETF Name': 'Vanguard Small Cap ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$24,678.09', 'YTD': '17.14%', 'Avg. Volume': '1,025,374', 'Previous Closing Price': '$154.61', 'Overall Rating': ''}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}]此解决方案利用while循环不断单击“下一步”按钮以浏览页面。这可以处理与站点提供的尽可能多的页面，而不是依赖于抓取前五个结果。