FMScraper/items.py

@@ -6,22 +6,13 @@
 # https://docs.scrapy.org/en/latest/topics/items.html
 
 import scrapy
-from scrapy.loader.processors import MapCompose
+
-import re
 
 class FmscraperItem(scrapy.Item):
     # define the fields for your item here like:
     # name = scrapy.Field()
     pass
 
-def clean_text(value):
-    yield value.strip()
-
-def clean_html_script(value):
-    description_re = re.match("([\w\W]+)(<script[\w\W]*</script>)([\w\W]+)",value)
-    description_full_post = description_re.group(1) + description_re.group(3)
-    yield description_full_post
-
 class show_Item(scrapy.Item):
     name = scrapy.Field()
     url_page = scrapy.Field()
@@ -29,18 +20,9 @@ class show_Item(scrapy.Item):
     tags = scrapy.Field()
 
 class episode_Item(scrapy.Item):
-    title = scrapy.Field(
+    title = scrapy.Field()
-            default = 'null',
+    description_lead = scrapy.Field()
-            input_processor = MapCompose(clean_text)
+    description_full = scrapy.Field()
-            )
-    description_lead = scrapy.Field(
-            default = 'null',
-            input_processor = MapCompose(clean_text)
-            )
-    description_full = scrapy.Field(
-            default = 'null',
-            input_processor = MapCompose(clean_html_script)
-            )
     url_file = scrapy.Field()
     url_page = scrapy.Field()
     date_diffusion = scrapy.Field()

FMScraper/spiders/get_episodes.py

@@ -2,37 +2,24 @@
 import scrapy
 from scrapy.loader import ItemLoader
 from scrapy.loader.processors import Join, MapCompose, TakeFirst
-from FMScraper.items import show_Item, episode_Item
+from FMScraper.items import episode_Item
 
 class GetEpisodesSpider(scrapy.Spider):
     name = 'get_episodes'
     allowed_domains = ['francemusique.fr']
     start_urls = ['https://www.francemusique.fr/emissions/l-experimentale']
 
-    show_fields = {
-            'name' :   '//h1/text()',
-            'url_feed'  :   '//div[@class="podcast-container rss"]//a/@href',
-            }
-
     episode_xpath = '//article[@class="diffusion"]'
     episode_fields = {
             'title' :   '//h1[@class="cover-diffusion-main-info-title"]/text()',
             'description_lead'  :   '//h2[@class="content-body-chapo"]/text()',
             'description_full' :    '//div[@class="content-body-article"]',
             'url_file'  :   '//div[@class="cover-diffusion-main-info"]//button[@class="replay-button playable"]/@data-url',
+#            'url_page'  :   '//iframe/@src',
             'date_diffusion'    :   '//div[@class="cover-diffusion-text-date"]/text()',
             }
 
     def parse(self, response):
-#        page_emission = scrapy.Selector(response)
-#        show_loader = ItemLoader(item=show_Item(), selector=page_emission)
-#        show_loader.default_output_processor = Join()
-#        for field, xpath in self.show_fields.items():
-#            show_loader.add_xpath(field, xpath)
-#        show_loader.add_value('url_page', response.url)
-
-#        yield show_loader.load_item()
-
         for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
             url_episode = response.urljoin(sel.extract())
             next_page = response.xpath('//link[@rel="next"]/@href')