README.md : Ajout modèle BDD

FMScraper/items.py

@@ -22,6 +22,10 @@ def clean_html_script(value):
     description_full_post = description_re.group(1) + description_re.group(3)
     yield description_full_post
 
+def clean_file_url(value):
+    url = re.match("(.+\.mp3)", value)
+    yield url.group(1)
+
 class show_Item(scrapy.Item):
     name = scrapy.Field()
     url_page = scrapy.Field()
@@ -38,10 +42,10 @@ class episode_Item(scrapy.Item):
             input_processor = MapCompose(clean_text)
             )
     description_full = scrapy.Field(
+    url_file = scrapy.Field(
             default = 'null',
-            input_processor = MapCompose(clean_html_script)
+            input_processor = MapCompose(clean_file_url)
             )
-    url_file = scrapy.Field()
     url_page = scrapy.Field()
     date_diffusion = scrapy.Field()
 

FMScraper/spiders/get_episodes.py

@@ -24,23 +24,24 @@ class GetEpisodesSpider(scrapy.Spider):
             }
 
     def parse(self, response):
-#        page_emission = scrapy.Selector(response)
+        page_emission = scrapy.Selector(response)
-#        show_loader = ItemLoader(item=show_Item(), selector=page_emission)
+        show_loader = ItemLoader(item=show_Item(), selector=page_emission)
-#        show_loader.default_output_processor = Join()
+        show_loader.default_output_processor = Join()
-#        for field, xpath in self.show_fields.items():
+        for field, xpath in self.show_fields.items():
-#            show_loader.add_xpath(field, xpath)
+            show_loader.add_xpath(field, xpath)
-#        show_loader.add_value('url_page', response.url)
+        show_loader.add_value('url_page', response.url)
 
-#        yield show_loader.load_item()
+        yield show_loader.load_item()
+        yield scrapy.Request(url=response.url, callback = self.parse_episodes)
 
+    def parse_episodes(self, response):
         for sel in response.xpath('//section[@class="emission-diffusions-list"]//a[@class="preview-list-element-link"]/@href'):
             url_episode = response.urljoin(sel.extract())
             next_page = response.xpath('//link[@rel="next"]/@href')
             yield scrapy.Request(url_episode, callback = self.parse_episode)
             if next_page:
                 next_url = response.urljoin(next_page.extract_first())
-                yield scrapy.Request(url=next_url)
+                yield scrapy.Request(url=next_url,callback = self.parse_episodes)
-
 
     def parse_episode(self, response):
         page_episode = scrapy.Selector(response)

README.md

@@ -11,3 +11,44 @@
 * Mettre à disposition un script pour créer la BDD
 * Automatiser le téléchargement des éléments vers un point donné
 * Récupérer les images illustrant les épisodes
+
+
+## Tables de la base RFScraper
+'''
+  CREATE TABLE station(
+      id serial PRIMARY KEY,
+      url text UNIQUE NOT NULL
+      );
+  
+  CREATE TABLE emission(
+      id serial PRIMARY KEY,
+      nom text NOT NULL,
+      url_page text NOT NULL,
+      url_rss text,
+      genre text,
+      producteur text,
+      station_id INTEGER REFERENCES station(id)
+      );
+  
+  CREATE TABLE episode(
+      id serial PRIMARY KEY,
+      titre text NOT NULL,
+      date_diffusion date,
+      description_lead text,
+      description_full text,
+      id_episode_site int UNIQUE NOT NULL,
+      emission_id INTEGER REFERENCES emission(id)
+      );
+  
+  CREATE TABLE media(
+      id serial PRIMARY KEY,
+      url_file text NOT NULL,
+      filename_orig text NOT NULL,
+      filename_local text,
+      size integer NOT NULL,
+      date_modif date NOT NULL,
+      md5 text,
+      duration real,
+      episode_id INTEGER REFERENCES episode(id)
+      );
+'''