Yohann
/
FMScraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Base fonctionnelle

master
Yohann Dedy 2019-11-26 23:33:46 +01:00
parent f33eda3ff1
commit 557c76c2ca
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
FMScraper/spiders/get_episodes.py
View File

@ -20,8 +20,8 @@ class GetEpisodesSpider(scrapy.Spider):
} }
def parse(self, response): def parse(self, response):
for sel in response.xpath('//section[@class="emission-diffusions-list"]'): for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
url_episode = response.urljoin(sel.xpath('.//a[@class="preview-list-element-link"]/@href').extract_first()) url_episode = response.urljoin(sel.extract())
yield scrapy.Request(url_episode, callback = self.parse_episode) yield scrapy.Request(url_episode, callback = self.parse_episode)
def parse_episode(self, response): def parse_episode(self, response):
@ -33,4 +33,6 @@ class GetEpisodesSpider(scrapy.Spider):
for field, xpath in self.episode_fields.items(): for field, xpath in self.episode_fields.items():
loader.add_xpath(field, xpath) loader.add_xpath(field, xpath)
loader.add_value('url_page', response.url)
yield loader.load_item() yield loader.load_item()