Yohann
/
FMScraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Iteration des pages suivantes fonctionnelle

master
Yohann Dedy 2019-11-27 00:05:34 +01:00
parent 557c76c2ca
commit ec40af2cce
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
FMScraper/spiders/get_episodes.py
View File

@ -22,7 +22,12 @@ class GetEpisodesSpider(scrapy.Spider):
def parse(self, response): def parse(self, response):
for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'): for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
url_episode = response.urljoin(sel.extract()) url_episode = response.urljoin(sel.extract())
next_page = response.xpath('//link[@rel="next"]/@href')
yield scrapy.Request(url_episode, callback = self.parse_episode) yield scrapy.Request(url_episode, callback = self.parse_episode)
if next_page:
next_url = response.urljoin(next_page.extract_first())
yield scrapy.Request(url=next_url)
def parse_episode(self, response): def parse_episode(self, response):
page_episode = scrapy.Selector(response) page_episode = scrapy.Selector(response)