Yohann
/
FMScraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: Yohann:f33eda3ff15c754aca4908995621796bc1b73b46
Branches Tags
Yohann:recuperation-emissions
Yohann:master
...
compare: Yohann:ec40af2cce4f2d0e3ef167ac0d9848c00b2b8a3e
Branches Tags
Yohann:recuperation-emissions
Yohann:master

2 Commits
f33eda3ff1 ... ec40af2cce

Author SHA1 Message Date
Yohann Dedy ec40af2cce Iteration des pages suivantes fonctionnelle 2019-11-27 00:05:34 +01:00
Yohann Dedy 557c76c2ca Base fonctionnelle 2019-11-26 23:33:46 +01:00
1 changed files with 9 additions and 2 deletions
Show Stats Expand all files Collapse all files

11
FMScraper/spiders/get_episodes.py
View File

@ -20,9 +20,14 @@ class GetEpisodesSpider(scrapy.Spider):
} }
def parse(self, response): def parse(self, response):
for sel in response.xpath('//section[@class="emission-diffusions-list"]'): for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
url_episode = response.urljoin(sel.xpath('.//a[@class="preview-list-element-link"]/@href').extract_first()) url_episode = response.urljoin(sel.extract())
next_page = response.xpath('//link[@rel="next"]/@href')
yield scrapy.Request(url_episode, callback = self.parse_episode) yield scrapy.Request(url_episode, callback = self.parse_episode)
if next_page:
next_url = response.urljoin(next_page.extract_first())
yield scrapy.Request(url=next_url)
def parse_episode(self, response): def parse_episode(self, response):
page_episode = scrapy.Selector(response) page_episode = scrapy.Selector(response)
@ -33,4 +38,6 @@ class GetEpisodesSpider(scrapy.Spider):
for field, xpath in self.episode_fields.items(): for field, xpath in self.episode_fields.items():
loader.add_xpath(field, xpath) loader.add_xpath(field, xpath)
loader.add_value('url_page', response.url)
yield loader.load_item() yield loader.load_item()