# -*- coding: utf-8 -*-
import scrapy
from douban.items import DoubanItem

class DoubanSpiderSpider(scrapy.Spider):
    #爬虫名字
    name = 'douban_spider'
    #允许的域名
    allowed_domains = ['movie.douban.com']
    #入口url, 扔到调度器里面去
    start_urls = ['http://movie.douban.com/top250']

    def parse(self, response):
        #循环电影的条目
        movie_list = response.xpath("//div[@class='article']//ol[@class='grid_view']//li")
        for i_item in movie_list:
            # item文件导进来
            douban_item = DoubanItem()
            # 写详细的xpath,进行数据解析
            douban_item['serial_number'] = i_item.xpath(
                ".//div[@class='item']//div[@class='pic']//em/text()").extract_first
            print(douban_item)

2018-08-17 15:23:28 [scrapy.utils.log] INFO: Scrapy 1.5.1 started (bot: douban)2018-08-17 15:23:28 [scrapy.utils.log] INFO: Versions: lxml 4.2.4.0, libxml2 2.9.5, cssselect 1.0.3, parsel 1.5.0, w3lib 1.19.0, Twisted 18.7.0, Python 3.7.0 (v3.7.0:1bf9cc5093, Jun 27 2018, 04:59:51) [MSC v.1914 64 bit (AMD64)], pyOpenSSL 18.0.0 (OpenSSL 1.1.0i  14 Aug 2018), cryptography 2.3.1, Platform Windows-10-10.0.17134-SP02018-08-17 15:23:28 [scrapy.crawler] INFO: Overridden settings: {'BOT_NAME': 'douban', 'DOWNLOAD_DELAY': 0.5, 'NEWSPIDER_MODULE': 'douban.spiders', 'SPIDER_MODULES': ['douban.spiders'], 'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}2018-08-17 15:23:28 [scrapy.middleware] INFO: Enabled extensions:['scrapy.extensions.corestats.CoreStats', 'scrapy.extensions.telnet.TelnetConsole', 'scrapy.extensions.logstats.LogStats']2018-08-17 15:23:28 [scrapy.middleware] INFO: Enabled downloader middlewares:['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', 'scrapy.downloadermiddlewares.retry.RetryMiddleware', 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', 'scrapy.downloadermiddlewares.stats.DownloaderStats']2018-08-17 15:23:28 [scrapy.middleware] INFO: Enabled spider middlewares:['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', 'scrapy.spidermiddlewares.referer.RefererMiddleware', 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', 'scrapy.spidermiddlewares.depth.DepthMiddleware']2018-08-17 15:23:28 [scrapy.middleware] INFO: Enabled item pipelines:[]2018-08-17 15:23:28 [scrapy.core.engine] INFO: Spider opened2018-08-17 15:23:28 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)2018-08-17 15:23:28 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:60232018-08-17 15:23:28 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://movie.douban.com/top250> from <GET http://movie.douban.com/top250>2018-08-17 15:23:29 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://movie.douban.com/top250> (referer: None)2018-08-17 15:23:29 [scrapy.core.engine] INFO: Closing spider (finished){'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='1'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='2'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='3'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='4'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='5'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='6'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='7'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='8'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='9'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='10'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='11'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='12'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='13'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='14'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='15'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='16'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='17'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='18'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='19'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='20'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='21'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='22'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='23'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='24'>]>}{'serial_number': <bound method SelectorList.get of [<Selector xpath=".//div[@class='item']//div[@class='pic']//em/text()" data='25'>]>}2018-08-17 15:23:29 [scrapy.statscollectors] INFO: Dumping Scrapy stats:{'downloader/request_bytes': 604, 'downloader/request_count': 2, 'downloader/request_method_count/GET': 2, 'downloader/response_bytes': 13243, 'downloader/response_count': 2, 'downloader/response_status_count/200': 1, 'downloader/response_status_count/301': 1, 'finish_reason': 'finished', 'finish_time': datetime.datetime(2018, 8, 17, 7, 23, 29, 444817), 'log_count/DEBUG': 3, 'log_count/INFO': 7, 'response_received_count': 1, 'scheduler/dequeued': 2, 'scheduler/dequeued/memory': 2, 'scheduler/enqueued': 2, 'scheduler/enqueued/memory': 2, 'start_time': datetime.datetime(2018, 8, 17, 7, 23, 28, 448175)}2018-08-17 15:23:29 [scrapy.core.engine] INFO: Spider closed (finished)Process finished with exit code 0