在settings.py中添加以下配置,再重新导出 FEED_EXPORT_ENCODING = 'UTF-8'
没有代码,也没有报错信息没法看
使用了循环
老师是将Linux中文件传到Windows中,你在cmd中说明你的开发环境是Windows,同样的命令就可以导出在项目目录下
这是我的代码 插入到mysql中 # -*- coding: utf-8 -*- import pymysql # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html class DoubanPipeline(object): def __init__(self): self.mydb = pymysql.connect(host="127.0.0.1",user="root",password="020202",db="douban",charset="utf8",use_unicode=True) self.cursor = self.mydb.cursor() def process_item(self, item, spider): insert_sql = """insert into movie(serial_number, movie_name, introduce, star, evaluate, describetion) values (%s, %s, %s, %s,%s,%s)""" # 使用`占位符 self.cursor.execute(insert_sql,(item['serial_number'],item['movie_name'],item['introduce'],item['star'],item['evaluate'],item['describetion'])) #用commit()才会提交到数据库 self.mydb.commit() def close_spider(self,spider): self.cursor.close() self.mydb.close()
自己登录 使用cookie
问题已解决,原来是本地的代码没有同步到虚拟机中的linux里。
import MySQLdb from douban.settings import mysql_host, mysql_port, mysql_user, mysql_passwd, mysql_db_name # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html class DoubanPipeline(object): def __init__(self): host = mysql_host port = mysql_port user = mysql_user passwd = mysql_passwd dbname = mysql_db_name self.mydb = MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, db=dbname, charset='utf8') self.cursor = self.mydb.cursor() def process_item(self, item, spider): data = dict(item) sql = "insert into top250 (serial_number, movie_name, introduce, star, evaluate, description) values (%s,%s,%s,%s,%s,%s)" self.cursor.execute(sql, (data['serial_number'], data['movie_name'], data['introduce'], data['star'], data['evaluate'], data['description'])) self.mydb.commit() return item
在pycharm左下terminal输入
scrapy crawl douban_spider -o xxx.csv 然后就可以在项目的spiders文件夹下面出现了
mongodb连接超时,试试看你MongoDB数据库在你本地是否能访问得到吧?
linux系统包管理命令yum,python包管理命令pip,直接安装即可,不需要提前下载安装包
需要贴出详细报错日志
mongodb服务未开启