diff --git a/CinemScraper/spiders/grabVideoData.py b/CinemScraper/spiders/grabVideoData.py index 05a1a3c..95a2c27 100644 --- a/CinemScraper/spiders/grabVideoData.py +++ b/CinemScraper/spiders/grabVideoData.py @@ -4,17 +4,21 @@ import scrapy class GrabvideodataSpider(scrapy.Spider): name = 'grabVideoData' - allowed_domains = ['http://www.cinematheque.fr/'] - start_urls = ['http://www.cinematheque.fr/video/1219.html'] + allowed_domains = ['cinematheque.fr'] + start_urls = ['http://www.cinematheque.fr/decouvrir.html'] def parse(self, response): + for lien in response.xpath('//a/@href[contains(.,"video")]/../..'): + url = response.urljoin(lien.css('a::attr(href)').extract_first()) + yield scrapy.Request(url, callback = self.parse_dir_content) + + def parse_dir_content(self, response): for page in response.css("div#content"): - yield { - 'titre' : page.css('h1::text').extract_first(), - 'sous-titre' : page.css('h1 span::text').extract_first(), - 'description' : page.css('.biographies p').extract(), - 'videoSrcUrl' : page.css('iframe::attr(src)').extract_first(), - 'tags' : page.css('.tag::text').extract() - } - - + yield { + 'titre' : page.css('h1::text').extract_first(), + 'sous-titre' : page.css('h1 span::text').extract_first(), + 'description' : page.css('.biographies p').extract(), + 'videoSrcUrl' : page.css('iframe::attr(src)').extract_first(), + 'tags' : page.css('.tag::text').extract() + } + diff --git a/CinemScraper/spiders/listeVideos.py b/CinemScraper/spiders/listeVideos.py deleted file mode 100644 index e25581b..0000000 --- a/CinemScraper/spiders/listeVideos.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -import scrapy - - -class ListevideosSpider(scrapy.Spider): - name = 'listeVideos' - allowed_domains = ['http://www.cinematheque.fr/'] - start_urls = ['http://www.cinematheque.fr/decouvrir.html'] - - def parse(self, response): - for lien in response.xpath('//a/@href[contains(.,"video")]/../..'): - yield { - 'adresse' : lien.css('a::attr(href)').extract(), - 'dateUpload' : lien.css('::attr(data-sort)').extract(), - - } - - diff --git a/CinemScraper/spiders/listeVideos.pyc b/CinemScraper/spiders/listeVideos.pyc deleted file mode 100644 index ca00c6a..0000000 Binary files a/CinemScraper/spiders/listeVideos.pyc and /dev/null differ