57 lines
2.4 KiB
Python
57 lines
2.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
import scrapy
|
|
from scrapy.loader import ItemLoader
|
|
from scrapy.loader.processors import Join, MapCompose, TakeFirst
|
|
from FMScraper.items import show_Item, episode_Item
|
|
|
|
class GetEpisodesSpider(scrapy.Spider):
|
|
name = 'get_episodes'
|
|
allowed_domains = ['francemusique.fr']
|
|
start_urls = ['https://www.francemusique.fr/emissions/l-experimentale']
|
|
|
|
show_fields = {
|
|
'name' : '//h1/text()',
|
|
'url_feed' : '//div[@class="podcast-container rss"]//a/@href',
|
|
}
|
|
|
|
episode_xpath = '//article[@class="diffusion"]'
|
|
episode_fields = {
|
|
'title' : '//h1[@class="cover-diffusion-main-info-title"]/text()',
|
|
'description_lead' : '//h2[@class="content-body-chapo"]/text()',
|
|
'description_full' : '//div[@class="content-body-article"]',
|
|
'url_file' : '//div[@class="cover-diffusion-main-info"]//button[@class="replay-button playable"]/@data-url',
|
|
'date_diffusion' : '//div[@class="cover-diffusion-text-date"]/text()',
|
|
}
|
|
|
|
def parse(self, response):
|
|
# page_emission = scrapy.Selector(response)
|
|
# show_loader = ItemLoader(item=show_Item(), selector=page_emission)
|
|
# show_loader.default_output_processor = Join()
|
|
# for field, xpath in self.show_fields.items():
|
|
# show_loader.add_xpath(field, xpath)
|
|
# show_loader.add_value('url_page', response.url)
|
|
|
|
# yield show_loader.load_item()
|
|
|
|
for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
|
|
url_episode = response.urljoin(sel.extract())
|
|
next_page = response.xpath('//link[@rel="next"]/@href')
|
|
yield scrapy.Request(url_episode, callback = self.parse_episode)
|
|
if next_page:
|
|
next_url = response.urljoin(next_page.extract_first())
|
|
yield scrapy.Request(url=next_url)
|
|
|
|
|
|
def parse_episode(self, response):
|
|
page_episode = scrapy.Selector(response)
|
|
for page in page_episode.xpath(self.episode_xpath):
|
|
loader = ItemLoader(item=episode_Item(), selector=page)
|
|
loader.default_output_processor = Join()
|
|
|
|
for field, xpath in self.episode_fields.items():
|
|
loader.add_xpath(field, xpath)
|
|
|
|
loader.add_value('url_page', response.url)
|
|
|
|
yield loader.load_item()
|