FMScraper/items.py

@@ -22,10 +22,6 @@ def clean_html_script(value):
     description_full_post = description_re.group(1) + description_re.group(3)
     yield description_full_post
 
-def clean_file_url(value):
-    url = re.match("(.+\.mp3)", value)
-    yield url.group(1)
-
 class show_Item(scrapy.Item):
     name = scrapy.Field()
     url_page = scrapy.Field()
@@ -42,10 +38,10 @@ class episode_Item(scrapy.Item):
             input_processor = MapCompose(clean_text)
             )
     description_full = scrapy.Field(
-    url_file = scrapy.Field(
             default = 'null',
-            input_processor = MapCompose(clean_file_url)
+            input_processor = MapCompose(clean_html_script)
             )
+    url_file = scrapy.Field()
     url_page = scrapy.Field()
     date_diffusion = scrapy.Field()
 

FMScraper/spiders/get_episodes.py

@@ -24,24 +24,23 @@ class GetEpisodesSpider(scrapy.Spider):
             }
 
     def parse(self, response):
-        page_emission = scrapy.Selector(response)
+#        page_emission = scrapy.Selector(response)
-        show_loader = ItemLoader(item=show_Item(), selector=page_emission)
+#        show_loader = ItemLoader(item=show_Item(), selector=page_emission)
-        show_loader.default_output_processor = Join()
+#        show_loader.default_output_processor = Join()
-        for field, xpath in self.show_fields.items():
+#        for field, xpath in self.show_fields.items():
-            show_loader.add_xpath(field, xpath)
+#            show_loader.add_xpath(field, xpath)
-        show_loader.add_value('url_page', response.url)
+#        show_loader.add_value('url_page', response.url)
 
-        yield show_loader.load_item()
+#        yield show_loader.load_item()
-        yield scrapy.Request(url=response.url, callback = self.parse_episodes)
 
-    def parse_episodes(self, response):
         for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
             url_episode = response.urljoin(sel.extract())
             next_page = response.xpath('//link[@rel="next"]/@href')
             yield scrapy.Request(url_episode, callback = self.parse_episode)
             if next_page:
                 next_url = response.urljoin(next_page.extract_first())
-                yield scrapy.Request(url=next_url,callback = self.parse_episodes)
+                yield scrapy.Request(url=next_url)
+
 
     def parse_episode(self, response):
         page_episode = scrapy.Selector(response)

README.md

@@ -11,44 +11,3 @@
 * Mettre à disposition un script pour créer la BDD
 * Automatiser le téléchargement des éléments vers un point donné
 * Récupérer les images illustrant les épisodes
-
-
-## Tables de la base RFScraper
-'''
-  CREATE TABLE station(
-      id serial PRIMARY KEY,
-      url text UNIQUE NOT NULL
-      );
-  
-  CREATE TABLE emission(
-      id serial PRIMARY KEY,
-      nom text NOT NULL,
-      url_page text NOT NULL,
-      url_rss text,
-      genre text,
-      producteur text,
-      station_id INTEGER REFERENCES station(id)
-      );
-  
-  CREATE TABLE episode(
-      id serial PRIMARY KEY,
-      titre text NOT NULL,
-      date_diffusion date,
-      description_lead text,
-      description_full text,
-      id_episode_site int UNIQUE NOT NULL,
-      emission_id INTEGER REFERENCES emission(id)
-      );
-  
-  CREATE TABLE media(
-      id serial PRIMARY KEY,
-      url_file text NOT NULL,
-      filename_orig text NOT NULL,
-      filename_local text,
-      size integer NOT NULL,
-      date_modif date NOT NULL,
-      md5 text,
-      duration real,
-      episode_id INTEGER REFERENCES episode(id)
-      );
-'''