利用python编写爬虫爬取淘宝奶粉部分数据.1-原创手记-慕课网

import requests
import re
import pymysql.cursors

def getHTMLText(url):
kv = {'User-Agent':'Mozilla/5.0'} #反反爬虫
try:
r = requests.get(url, headers = kv, timeout = 30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
print("error1")

def parsePage(ilt,html):
try:

从字符串类型的html中用正则表达式查找相关数据项

    price = re.findall(r'\"view_price\"\:\"[\d\.]*\"',html)
    title = re.findall(r'\"raw_title\"\:\".*?\"',html)
    place = re.findall(r'"item_loc":".*?"',html)
    for i in range(len(price)):
        #从查找的相关数据项中分离数据，可以在另一交互界面检查分离数据的数据类型
        pprice = eval(price[i].split(':')[1])
        ttitle = eval(title[i].split(':')[1])
        pplace = eval(place[i].split(':')[1])
        ilt.append([pprice,ttitle,pplace])
except:
        print("error2")

def printGoodsList(ilt):
tplt = "{:^4}\t{:8}\t{:^26}\t{:^80}"
print(tplt.format("序号","价格","商品名称","所属地"))
count = 0
num = 0
for g in ilt:
count = count +1
print(tplt.format(count,g[0],g[1],g[2]))

连接数据库（表事先已经建立好）

connect = pymysql.Connect(host="localhost",user="root",password="1984ispb",db="TianmaoMilkPowder",port=3306,charset="utf8")
cursor = connect.cursor()
for g in ilt:
    num =num + 1
    sql = "insert into Powderdata(No,Price,goodsName,Place) values ('%d','%s','%s','%s')"
    data = (num,g[0],g[1],g[2])
    try:
        #执行插入操作
        cursor.execute(sql % data)
        connect.commit()
    except:
        print("错误")
connect.commit()
cursor.close()
print('成功插入', cursor.rowcount, '条数据')

def main():
godds = '奶粉'
depth = 10
start_url = https://s.taobao.com/search?q=goods
infoList = []
for i in range(depth):
try:

换页操作

        url = start_url + '&s=' + str(44*i)
        html = getHTMLText(url)
        parsePage(infoList,html)
    except:
        continue
printGoodsList(infoList)

main()