我试图抓取这个网站,但我的蜘蛛返回空的 json,我无法理解问题出在哪里。我的代码和/或我的方法有问题吗?谢谢
import scrapy
import json
class SrealitySpider(scrapy.Spider):
name = 'sreality'
allowed_domains = ['www.sreality.cz/en']
def start_requests(self):
yield scrapy.Request(url="https://www.sreality.cz/api/en/v2/estates?category_main_cb=1&category_type_cb=2&locality_region_id=10&per_page=20&tms=1548939428469",
callback=self.parse_id)
def parse_id(self, response):
data = json.loads(response.body)
estates = data.get("_embedded").get("estates")
for estate in estates:
yield scrapy.Request(url="https://www.sreality.cz/api{0}?tms=1548942301694".format(estate.get("_links").get("self").get("href")), callback=self.parse)
def parse(self, response):
estate = json.loads(response.body)
yield {
"lat": estate.get("map").get("lat"),
"lon": estate.get("map").get("lon"),
"title": estate.get("name").get("value").replace(u'\xa0', u' '),
"address": estate.get("locality").get("value"),
"Price (czk)": estate.get("price_czk").get("value").replace(u'\xa0', u' '),
"nearby (m)": {prox.get("name"): prox.get("distance") for prox in estate.get("poi")},
"attributes": {attrib.get("name"): attrib.get("value") for attrib in estate.get("items")}
}
相关分类