Yohann
/
FMScraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Récupération infos émission avant épisode

master
Yohann Dedy 2019-12-01 21:59:56 +01:00
parent 11fd8d61c3
commit 1b9ac23e85
1 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
FMScraper/spiders/get_episodes.py
View File

@ -24,23 +24,24 @@ class GetEpisodesSpider(scrapy.Spider):
} }
def parse(self, response): def parse(self, response):
# page_emission = scrapy.Selector(response) page_emission = scrapy.Selector(response)
# show_loader = ItemLoader(item=show_Item(), selector=page_emission) show_loader = ItemLoader(item=show_Item(), selector=page_emission)
# show_loader.default_output_processor = Join() show_loader.default_output_processor = Join()
# for field, xpath in self.show_fields.items(): for field, xpath in self.show_fields.items():
# show_loader.add_xpath(field, xpath) show_loader.add_xpath(field, xpath)
# show_loader.add_value('url_page', response.url) show_loader.add_value('url_page', response.url)
# yield show_loader.load_item() yield show_loader.load_item()
yield scrapy.Request(url=response.url, callback = self.parse_episodes)
def parse_episodes(self, response):
for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'): for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
url_episode = response.urljoin(sel.extract()) url_episode = response.urljoin(sel.extract())
next_page = response.xpath('//link[@rel="next"]/@href') next_page = response.xpath('//link[@rel="next"]/@href')
yield scrapy.Request(url_episode, callback = self.parse_episode) yield scrapy.Request(url_episode, callback = self.parse_episode)
if next_page: if next_page:
next_url = response.urljoin(next_page.extract_first()) next_url = response.urljoin(next_page.extract_first())
yield scrapy.Request(url=next_url) yield scrapy.Request(url=next_url,callback = self.parse_episodes)
def parse_episode(self, response): def parse_episode(self, response):
page_episode = scrapy.Selector(response) page_episode = scrapy.Selector(response)