From ec40af2cce4f2d0e3ef167ac0d9848c00b2b8a3e Mon Sep 17 00:00:00 2001 From: Yohann Dedy Date: Wed, 27 Nov 2019 00:05:34 +0100 Subject: [PATCH] Iteration des pages suivantes fonctionnelle --- FMScraper/spiders/get_episodes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/FMScraper/spiders/get_episodes.py b/FMScraper/spiders/get_episodes.py index 781e52c..3f461a8 100644 --- a/FMScraper/spiders/get_episodes.py +++ b/FMScraper/spiders/get_episodes.py @@ -22,7 +22,12 @@ class GetEpisodesSpider(scrapy.Spider): def parse(self, response): for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'): url_episode = response.urljoin(sel.extract()) + next_page = response.xpath('//link[@rel="next"]/@href') yield scrapy.Request(url_episode, callback = self.parse_episode) + if next_page: + next_url = response.urljoin(next_page.extract_first()) + yield scrapy.Request(url=next_url) + def parse_episode(self, response): page_episode = scrapy.Selector(response)