Yohann
/
FMScraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Préparatifs scraping des infos de l'émission

master
Yohann Dedy 2019-11-29 01:13:26 +01:00
parent 21aa23b892
commit ae81fac37e
1 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
FMScraper/spiders/get_episodes.py
View File

@ -2,24 +2,37 @@
import scrapy import scrapy
from scrapy.loader import ItemLoader from scrapy.loader import ItemLoader
from scrapy.loader.processors import Join, MapCompose, TakeFirst from scrapy.loader.processors import Join, MapCompose, TakeFirst
from FMScraper.items import episode_Item from FMScraper.items import show_Item, episode_Item
class GetEpisodesSpider(scrapy.Spider): class GetEpisodesSpider(scrapy.Spider):
name = 'get_episodes' name = 'get_episodes'
allowed_domains = ['francemusique.fr'] allowed_domains = ['francemusique.fr']
start_urls = ['https://www.francemusique.fr/emissions/l-experimentale'] start_urls = ['https://www.francemusique.fr/emissions/l-experimentale']
show_fields = {
'name' : '//h1/text()',
'url_feed' : '//div[@class="podcast-container rss"]//a/@href',
}
episode_xpath = '//article[@class="diffusion"]' episode_xpath = '//article[@class="diffusion"]'
episode_fields = { episode_fields = {
'title' : '//h1[@class="cover-diffusion-main-info-title"]/text()', 'title' : '//h1[@class="cover-diffusion-main-info-title"]/text()',
'description_lead' : '//h2[@class="content-body-chapo"]/text()', 'description_lead' : '//h2[@class="content-body-chapo"]/text()',
'description_full' : '//div[@class="content-body-article"]', 'description_full' : '//div[@class="content-body-article"]',
'url_file' : '//div[@class="cover-diffusion-main-info"]//button[@class="replay-button playable"]/@data-url', 'url_file' : '//div[@class="cover-diffusion-main-info"]//button[@class="replay-button playable"]/@data-url',
# 'url_page' : '//iframe/@src',
'date_diffusion' : '//div[@class="cover-diffusion-text-date"]/text()', 'date_diffusion' : '//div[@class="cover-diffusion-text-date"]/text()',
} }
def parse(self, response): def parse(self, response):
# page_emission = scrapy.Selector(response)
# show_loader = ItemLoader(item=show_Item(), selector=page_emission)
# show_loader.default_output_processor = Join()
# for field, xpath in self.show_fields.items():
# show_loader.add_xpath(field, xpath)
# show_loader.add_value('url_page', response.url)
# yield show_loader.load_item()
for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'): for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
url_episode = response.urljoin(sel.extract()) url_episode = response.urljoin(sel.extract())
next_page = response.xpath('//link[@rel="next"]/@href') next_page = response.xpath('//link[@rel="next"]/@href')