diff --git a/FMScraper/spiders/get_episodes.py b/FMScraper/spiders/get_episodes.py index 3f461a8..6993255 100644 --- a/FMScraper/spiders/get_episodes.py +++ b/FMScraper/spiders/get_episodes.py @@ -2,24 +2,37 @@ import scrapy from scrapy.loader import ItemLoader from scrapy.loader.processors import Join, MapCompose, TakeFirst -from FMScraper.items import episode_Item +from FMScraper.items import show_Item, episode_Item class GetEpisodesSpider(scrapy.Spider): name = 'get_episodes' allowed_domains = ['francemusique.fr'] start_urls = ['https://www.francemusique.fr/emissions/l-experimentale'] + show_fields = { + 'name' : '//h1/text()', + 'url_feed' : '//div[@class="podcast-container rss"]//a/@href', + } + episode_xpath = '//article[@class="diffusion"]' episode_fields = { 'title' : '//h1[@class="cover-diffusion-main-info-title"]/text()', 'description_lead' : '//h2[@class="content-body-chapo"]/text()', 'description_full' : '//div[@class="content-body-article"]', 'url_file' : '//div[@class="cover-diffusion-main-info"]//button[@class="replay-button playable"]/@data-url', -# 'url_page' : '//iframe/@src', 'date_diffusion' : '//div[@class="cover-diffusion-text-date"]/text()', } def parse(self, response): +# page_emission = scrapy.Selector(response) +# show_loader = ItemLoader(item=show_Item(), selector=page_emission) +# show_loader.default_output_processor = Join() +# for field, xpath in self.show_fields.items(): +# show_loader.add_xpath(field, xpath) +# show_loader.add_value('url_page', response.url) + +# yield show_loader.load_item() + for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'): url_episode = response.urljoin(sel.extract()) next_page = response.xpath('//link[@rel="next"]/@href')