30 lines
723 B
Python
30 lines
723 B
Python
import peewee
|
|
from itemadapter import ItemAdapter
|
|
|
|
from data_collector.models import Event, initialize_db
|
|
|
|
|
|
class DataCollectorPipeline:
|
|
|
|
def open_spider(self, spider):
|
|
self.db = initialize_db()
|
|
|
|
def process_item(self, item, spider):
|
|
event = Event(
|
|
title=item["title"],
|
|
start_datetime=item["start_datetime"],
|
|
end_datetime=item["end_datetime"],
|
|
url=item["url"],
|
|
location=item["location"],
|
|
processed=False,
|
|
)
|
|
try:
|
|
event.save()
|
|
except peewee.IntegrityError:
|
|
# Event was added before
|
|
pass
|
|
|
|
return item
|
|
|
|
def close_spider(self, spider):
|
|
self.db.close()
|