94 lines
3.3 KiB
Python
94 lines
3.3 KiB
Python
import os
|
|
import sqlite3
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
|
|
DB_PATH = os.getenv("DATABASE_PATH", "data/what2do.db")
|
|
|
|
SCHEMA = """
|
|
PRAGMA journal_mode=WAL;
|
|
PRAGMA foreign_keys=ON;
|
|
CREATE TABLE IF NOT EXISTS sources (
|
|
id INTEGER PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
url TEXT NOT NULL UNIQUE,
|
|
venue_name TEXT NOT NULL,
|
|
venue_address TEXT NOT NULL,
|
|
latitude REAL,
|
|
longitude REAL,
|
|
timezone TEXT NOT NULL DEFAULT 'Australia/Sydney',
|
|
strategy_json TEXT NOT NULL DEFAULT '{}',
|
|
item_selector TEXT NOT NULL,
|
|
title_selector TEXT NOT NULL,
|
|
date_selector TEXT NOT NULL,
|
|
time_selector TEXT NOT NULL DEFAULT '',
|
|
link_selector TEXT NOT NULL DEFAULT '',
|
|
description_selector TEXT NOT NULL DEFAULT '',
|
|
location_selector TEXT NOT NULL DEFAULT '',
|
|
date_format TEXT NOT NULL DEFAULT '',
|
|
recurring_weekly INTEGER NOT NULL DEFAULT 0,
|
|
enabled INTEGER NOT NULL DEFAULT 1,
|
|
crawl_interval_minutes INTEGER NOT NULL DEFAULT 360,
|
|
last_crawled_at TEXT,
|
|
last_status TEXT,
|
|
last_error TEXT,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
CREATE TABLE IF NOT EXISTS events (
|
|
id INTEGER PRIMARY KEY,
|
|
source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
|
external_key TEXT NOT NULL,
|
|
title TEXT NOT NULL,
|
|
description TEXT NOT NULL DEFAULT '',
|
|
starts_at TEXT NOT NULL,
|
|
ends_at TEXT,
|
|
venue_name TEXT NOT NULL,
|
|
address TEXT NOT NULL,
|
|
latitude REAL,
|
|
longitude REAL,
|
|
event_url TEXT NOT NULL,
|
|
hidden INTEGER NOT NULL DEFAULT 0,
|
|
last_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE(source_id, external_key)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_events_starts_at ON events(starts_at);
|
|
CREATE INDEX IF NOT EXISTS idx_events_source ON events(source_id);
|
|
CREATE TABLE IF NOT EXISTS geocode_cache (
|
|
query TEXT PRIMARY KEY,
|
|
latitude REAL,
|
|
longitude REAL,
|
|
display_name TEXT,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
"""
|
|
|
|
@contextmanager
|
|
def connect():
|
|
path = Path(DB_PATH)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
db = sqlite3.connect(path)
|
|
db.row_factory = sqlite3.Row
|
|
db.execute("PRAGMA foreign_keys=ON")
|
|
try:
|
|
yield db
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
def init_db():
|
|
with connect() as db:
|
|
db.executescript(SCHEMA)
|
|
event_columns = {row["name"] for row in db.execute("PRAGMA table_info(events)")}
|
|
if "hidden" not in event_columns:
|
|
db.execute("ALTER TABLE events ADD COLUMN hidden INTEGER NOT NULL DEFAULT 0")
|
|
source_columns = {row["name"] for row in db.execute("PRAGMA table_info(sources)")}
|
|
if "strategy_json" not in source_columns:
|
|
db.execute("ALTER TABLE sources ADD COLUMN strategy_json TEXT NOT NULL DEFAULT '{}'")
|
|
db.execute("""UPDATE sources SET strategy_json=json_object(
|
|
'mode','html','request_url',url,'method','GET','response_type','html',
|
|
'item_selector',item_selector,'title_selector',title_selector,'date_selector',date_selector,
|
|
'time_selector',time_selector,'link_selector',link_selector,'description_selector',description_selector,
|
|
'location_selector',location_selector,'date_format',date_format,'recurring_weekly',recurring_weekly
|
|
) WHERE strategy_json='{}' OR strategy_json=''""")
|