Yohann
/
FMScraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: Yohann:ec40af2cce4f2d0e3ef167ac0d9848c00b2b8a3e
Branches Tags
Yohann:recuperation-emissions
Yohann:master
..
compare: Yohann:f33eda3ff15c754aca4908995621796bc1b73b46
Branches Tags
Yohann:recuperation-emissions
Yohann:master

No commits in common. "ec40af2cce4f2d0e3ef167ac0d9848c00b2b8a3e" and "f33eda3ff15c754aca4908995621796bc1b73b46" have entirely different histories.
ec40af2cce ... f33eda3ff1

1 changed files with 2 additions and 9 deletions
Show Stats Expand all files Collapse all files

11
FMScraper/spiders/get_episodes.py
View File

@ -20,14 +20,9 @@ class GetEpisodesSpider(scrapy.Spider):
} }
def parse(self, response): def parse(self, response):
for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'): for sel in response.xpath('//section[@class="emission-diffusions-list"]'):
url_episode = response.urljoin(sel.extract()) url_episode = response.urljoin(sel.xpath('.//a[@class="preview-list-element-link"]/@href').extract_first())
next_page = response.xpath('//link[@rel="next"]/@href')
yield scrapy.Request(url_episode, callback = self.parse_episode) yield scrapy.Request(url_episode, callback = self.parse_episode)
if next_page:
next_url = response.urljoin(next_page.extract_first())
yield scrapy.Request(url=next_url)
def parse_episode(self, response): def parse_episode(self, response):
page_episode = scrapy.Selector(response) page_episode = scrapy.Selector(response)
@ -38,6 +33,4 @@ class GetEpisodesSpider(scrapy.Spider):
for field, xpath in self.episode_fields.items(): for field, xpath in self.episode_fields.items():
loader.add_xpath(field, xpath) loader.add_xpath(field, xpath)
loader.add_value('url_page', response.url)
yield loader.load_item() yield loader.load_item()