Préparatifs scraping des infos de l'émission

FMScraper/items.py

@@ -6,13 +6,22 @@
 # https://docs.scrapy.org/en/latest/topics/items.html
 
 import scrapy
-
+from scrapy.loader.processors import MapCompose
+import re
 
 class FmscraperItem(scrapy.Item):
     # define the fields for your item here like:
     # name = scrapy.Field()
     pass
 
+def clean_text(value):
+    yield value.strip()
+
+def clean_html_script(value):
+    description_re = re.match("([\w\W]+)(<script[\w\W]*</script>)([\w\W]+)",value)
+    description_full_post = description_re.group(1) + description_re.group(3)
+    yield description_full_post
+
 class show_Item(scrapy.Item):
     name = scrapy.Field()
     url_page = scrapy.Field()
@@ -20,9 +29,18 @@ class show_Item(scrapy.Item):
     tags = scrapy.Field()
 
 class episode_Item(scrapy.Item):
-    title = scrapy.Field()
-    description_lead = scrapy.Field()
-    description_full = scrapy.Field()
+    title = scrapy.Field(
+            default = 'null',
+            input_processor = MapCompose(clean_text)
+            )
+    description_lead = scrapy.Field(
+            default = 'null',
+            input_processor = MapCompose(clean_text)
+            )
+    description_full = scrapy.Field(
+            default = 'null',
+            input_processor = MapCompose(clean_html_script)
+            )
     url_file = scrapy.Field()
     url_page = scrapy.Field()
     date_diffusion = scrapy.Field()

FMScraper/spiders/get_episodes.py

@@ -2,24 +2,37 @@
 import scrapy
 from scrapy.loader import ItemLoader
 from scrapy.loader.processors import Join, MapCompose, TakeFirst
-from FMScraper.items import episode_Item
+from FMScraper.items import show_Item, episode_Item
 
 class GetEpisodesSpider(scrapy.Spider):
     name = 'get_episodes'
     allowed_domains = ['francemusique.fr']
     start_urls = ['https://www.francemusique.fr/emissions/l-experimentale']
 
+    show_fields = {
+            'name' :   '//h1/text()',
+            'url_feed'  :   '//div[@class="podcast-container rss"]//a/@href',
+            }
+
     episode_xpath = '//article[@class="diffusion"]'
     episode_fields = {
             'title' :   '//h1[@class="cover-diffusion-main-info-title"]/text()',
             'description_lead'  :   '//h2[@class="content-body-chapo"]/text()',
             'description_full' :    '//div[@class="content-body-article"]',
             'url_file'  :   '//div[@class="cover-diffusion-main-info"]//button[@class="replay-button playable"]/@data-url',
-#            'url_page'  :   '//iframe/@src',
             'date_diffusion'    :   '//div[@class="cover-diffusion-text-date"]/text()',
             }
 
     def parse(self, response):
+#        page_emission = scrapy.Selector(response)
+#        show_loader = ItemLoader(item=show_Item(), selector=page_emission)
+#        show_loader.default_output_processor = Join()
+#        for field, xpath in self.show_fields.items():
+#            show_loader.add_xpath(field, xpath)
+#        show_loader.add_value('url_page', response.url)
+
+#        yield show_loader.load_item()
+
         for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
             url_episode = response.urljoin(sel.extract())
             next_page = response.xpath('//link[@rel="next"]/@href')