Yohann
/
FMScraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Ajout de préprocesseurs pour les items

master
Yohann Dedy 2019-11-28 23:59:01 +01:00
parent ec40af2cce
commit 280ad9842b
1 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
FMScraper/items.py
View File

@ -6,13 +6,22 @@
# https://docs.scrapy.org/en/latest/topics/items.html # https://docs.scrapy.org/en/latest/topics/items.html
import scrapy import scrapy
from scrapy.loader.processors import MapCompose
import re
class FmscraperItem(scrapy.Item): class FmscraperItem(scrapy.Item):
# define the fields for your item here like: # define the fields for your item here like:
# name = scrapy.Field() # name = scrapy.Field()
pass pass
def clean_text(value):
yield value.strip()
def clean_html_script(value):
description_re = re.match("([\w\W]+)(<script[\w\W]*</script>)([\w\W]+)",value)
description_full_post = description_re.group(1) + description_re.group(3)
yield description_full_post
class show_Item(scrapy.Item): class show_Item(scrapy.Item):
name = scrapy.Field() name = scrapy.Field()
url_page = scrapy.Field() url_page = scrapy.Field()
@ -21,8 +30,14 @@ class show_Item(scrapy.Item):
class episode_Item(scrapy.Item): class episode_Item(scrapy.Item):
title = scrapy.Field() title = scrapy.Field()
description_lead = scrapy.Field() description_lead = scrapy.Field(
description_full = scrapy.Field() default = 'null',
input_processor = MapCompose(clean_text)
)
description_full = scrapy.Field(
default = 'null',
input_processor = MapCompose(clean_html_script)
)
url_file = scrapy.Field() url_file = scrapy.Field()
url_page = scrapy.Field() url_page = scrapy.Field()
date_diffusion = scrapy.Field() date_diffusion = scrapy.Field()