Scrapy：如何将 url_id 与抓取的数据一起存储

Question

from scrapy import Spider, Request

from selenium import webdriver

class MySpider(Spider):

    name = "my_spider"

    def __init__(self):

        self.browser = webdriver.Chrome(executable_path='E:/chromedriver')

        self.browser.set_page_load_timeout(100)

    def closed(self,spider):

        print("spider closed")

        self.browser.close()

    def start_requests(self):

        start_urls = []

        with open("target_urls.txt", 'r', encoding='utf-8') as f:

            for line in f:

                url_id, url = line.split('\t\t')

                start_urls.append(url)

        for url in start_urls:

            yield Request(url=url, callback=self.parse)

    def parse(self, response):

        yield {

            'target_url': response.url,

            'comments': response.xpath('//div[@class="comments"]//em//text()').extract()

        }

以上是我的scrapy代码。我scrapy crawl my_spider -o comments.json用来运行爬虫。

您可能会注意到，对于每个 my url，都有一个url_id与之相关的独特之处。如何将每个抓取的结果与url_id. 理想情况下，我想url_id将 .yield 输出结果存储在comments.json.

Answer 1

meta例如，尝试传入参数。我对你的代码做了一些更新：def start_requests(self):    with open("target_urls.txt", 'r', encoding='utf-8') as f:        for line in f:            url_id, url = line.split('\t\t')            yield Request(url, self.parse, meta={'url_id': url_id, 'original_url': url})def parse(self, response):    yield {        'target_url': response.meta['original_url'],        'url_id': response.meta['url_id'],        'comments': response.xpath('//div[@class="comments"]//em//text()').extract()    }

Answer 2

回答问题和评论，尝试这样的事情：from scrapy import Spider, Requestfrom selenium import webdriverclass MySpider(Spider):    name = "my_spider"    def __init__(self):        self.browser = webdriver.Chrome(executable_path='E:/chromedriver')        self.browser.set_page_load_timeout(100)    def closed(self,spider):        print("spider closed")        self.browser.close()    def start_requests(self):        with open("target_urls.txt", 'r', encoding='utf-8') as f:            for line in f:                url_id, url = line.split('\t\t')                yield Request(url=url, callback=self.parse, meta={'url_id':url_id,'url':url})    def parse(self, response):        yield {        'target_url': response.meta['url'],        'comments': response.xpath('//div[@class="comments"]//em//text()').extract(),        'url_id':response.meta['url_id']    }正如上一个答案中所说，您可以使用 META ( http://scrapingauthority.com/scrapy-meta )在各种方法之间传递参数。