72 lines
1.9 KiB
Python
72 lines
1.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Define here the models for your scraped items
|
|
#
|
|
# See documentation in:
|
|
# https://docs.scrapy.org/en/latest/topics/items.html
|
|
|
|
import scrapy
|
|
from scrapy.loader.processors import MapCompose
|
|
import re
|
|
|
|
class FmscraperItem(scrapy.Item):
|
|
# define the fields for your item here like:
|
|
# name = scrapy.Field()
|
|
pass
|
|
|
|
def clean_text(value):
|
|
yield value.strip()
|
|
|
|
def clean_html_script(value):
|
|
description_re = re.match("([\w\W]+)(<script[\w\W]*</script>)?([\w\W]+)",value)
|
|
description_full_post = description_re.group(1) + description_re.group(3)
|
|
yield description_full_post
|
|
|
|
def clean_file_url(value):
|
|
url = re.match("(.+\.mp3)", value)
|
|
yield url.group(1)
|
|
|
|
def clean_emission_url(value):
|
|
url = re.match("(.+)(\?.+$)?", value)
|
|
yield url.group(1)
|
|
|
|
def extract_id_episode(value):
|
|
id_episode = re.search("-([\d]+$)", value)
|
|
yield id_episode.group(1)
|
|
|
|
class show_Item(scrapy.Item):
|
|
name = scrapy.Field()
|
|
url_page = scrapy.Field()
|
|
url_feed = scrapy.Field()
|
|
tags = scrapy.Field()
|
|
|
|
class episode_Item(scrapy.Item):
|
|
url_emission = scrapy.Field(
|
|
default = 'null',
|
|
input_processor = MapCompose(clean_emission_url)
|
|
)
|
|
title = scrapy.Field(
|
|
default = 'null',
|
|
input_processor = MapCompose(clean_text)
|
|
)
|
|
id_episode = scrapy.Field(
|
|
default = 'null',
|
|
input_processor = MapCompose(extract_id_episode)
|
|
)
|
|
description_lead = scrapy.Field(
|
|
default = 'null',
|
|
input_processor = MapCompose(clean_text)
|
|
)
|
|
description_full = scrapy.Field(
|
|
default = 'null',
|
|
input_processor = MapCompose(clean_html_script)
|
|
)
|
|
url_file = scrapy.Field(
|
|
default = 'null',
|
|
input_processor = MapCompose(clean_file_url)
|
|
)
|
|
url_page = scrapy.Field()
|
|
date_diffusion = scrapy.Field()
|
|
file_size= scrapy.Field()
|
|
|