FMScraper/FMScraper/items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy
from scrapy.loader.processors import MapCompose
import re

class FmscraperItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass

def clean_text(value):
    yield value.strip()

def clean_html_script(value):
    description_re = re.match("([\w\W]+)(<script[\w\W]*</script>)?([\w\W]+)",value)
    description_full_post = description_re.group(1) + description_re.group(3)
    yield description_full_post

def clean_file_url(value):
    url = re.match("(.+\.mp3)", value)
    yield url.group(1)

def clean_emission_url(value):
    url = re.match("(.+)(\?.+$)?", value)
    yield url.group(1)

def extract_id_episode(value):
    id_episode = re.search("-([\d]+$)", value)
    yield id_episode.group(1)

class show_Item(scrapy.Item):
    name = scrapy.Field()
    url_page = scrapy.Field()
    url_feed = scrapy.Field()
    tags = scrapy.Field()

class episode_Item(scrapy.Item):
    url_emission = scrapy.Field(
            default = 'null',
            input_processor = MapCompose(clean_emission_url)
            )
    title = scrapy.Field(
            default = 'null',
            input_processor = MapCompose(clean_text)
            )
    id_episode = scrapy.Field(
            default = 'null',
            input_processor = MapCompose(extract_id_episode)
            )
    description_lead = scrapy.Field(
            default = 'null',
            input_processor = MapCompose(clean_text)
            )
    description_full = scrapy.Field(
            default = 'null',
            input_processor = MapCompose(clean_html_script)
            )
    url_file = scrapy.Field(
            default = 'null',
            input_processor = MapCompose(clean_file_url)
            )
    url_page = scrapy.Field()
    date_diffusion = scrapy.Field()
    file_size= scrapy.Field()
    file_last_modified = scrapy.Field()