import os import sqlite3 from contextlib import contextmanager from pathlib import Path DB_PATH = os.getenv("DATABASE_PATH", "data/what2do.db") SCHEMA = """ PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; CREATE TABLE IF NOT EXISTS sources ( id INTEGER PRIMARY KEY, name TEXT NOT NULL, url TEXT NOT NULL UNIQUE, venue_name TEXT NOT NULL, venue_address TEXT NOT NULL, latitude REAL, longitude REAL, timezone TEXT NOT NULL DEFAULT 'Australia/Sydney', strategy_json TEXT NOT NULL DEFAULT '{}', item_selector TEXT NOT NULL, title_selector TEXT NOT NULL, date_selector TEXT NOT NULL, time_selector TEXT NOT NULL DEFAULT '', link_selector TEXT NOT NULL DEFAULT '', description_selector TEXT NOT NULL DEFAULT '', location_selector TEXT NOT NULL DEFAULT '', date_format TEXT NOT NULL DEFAULT '', recurring_weekly INTEGER NOT NULL DEFAULT 0, enabled INTEGER NOT NULL DEFAULT 1, crawl_interval_minutes INTEGER NOT NULL DEFAULT 360, last_crawled_at TEXT, last_status TEXT, last_error TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, external_key TEXT NOT NULL, title TEXT NOT NULL, description TEXT NOT NULL DEFAULT '', starts_at TEXT NOT NULL, ends_at TEXT, venue_name TEXT NOT NULL, address TEXT NOT NULL, latitude REAL, longitude REAL, event_url TEXT NOT NULL, hidden INTEGER NOT NULL DEFAULT 0, last_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(source_id, external_key) ); CREATE INDEX IF NOT EXISTS idx_events_starts_at ON events(starts_at); CREATE INDEX IF NOT EXISTS idx_events_source ON events(source_id); CREATE TABLE IF NOT EXISTS geocode_cache ( query TEXT PRIMARY KEY, latitude REAL, longitude REAL, display_name TEXT, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); """ @contextmanager def connect(): path = Path(DB_PATH) path.parent.mkdir(parents=True, exist_ok=True) db = sqlite3.connect(path) db.row_factory = sqlite3.Row db.execute("PRAGMA foreign_keys=ON") try: yield db db.commit() finally: db.close() def init_db(): with connect() as db: db.executescript(SCHEMA) event_columns = {row["name"] for row in db.execute("PRAGMA table_info(events)")} if "hidden" not in event_columns: db.execute("ALTER TABLE events ADD COLUMN hidden INTEGER NOT NULL DEFAULT 0") source_columns = {row["name"] for row in db.execute("PRAGMA table_info(sources)")} if "strategy_json" not in source_columns: db.execute("ALTER TABLE sources ADD COLUMN strategy_json TEXT NOT NULL DEFAULT '{}'") db.execute("""UPDATE sources SET strategy_json=json_object( 'mode','html','request_url',url,'method','GET','response_type','html', 'item_selector',item_selector,'title_selector',title_selector,'date_selector',date_selector, 'time_selector',time_selector,'link_selector',link_selector,'description_selector',description_selector, 'location_selector',location_selector,'date_format',date_format,'recurring_weekly',recurring_weekly ) WHERE strategy_json='{}' OR strategy_json=''""")