623 lines
37 KiB
Python
623 lines
37 KiB
Python
import asyncio
|
||
import json
|
||
import math
|
||
import os
|
||
import secrets
|
||
import threading
|
||
from calendar import Calendar
|
||
from contextlib import asynccontextmanager
|
||
from datetime import datetime, timedelta, timezone
|
||
from urllib.parse import urljoin
|
||
|
||
import httpx
|
||
from fastapi import FastAPI, HTTPException, Query, Request
|
||
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
|
||
from fastapi.staticfiles import StaticFiles
|
||
from fastapi.templating import Jinja2Templates
|
||
from bs4 import BeautifulSoup
|
||
|
||
from .crawler import CrawlError, _http_client, detect_rules, external_key, fetch_html, infer_page_fields
|
||
from .db import connect, init_db
|
||
from .strategies import execute_strategy, parse_strategy
|
||
|
||
TEMPLATES = Jinja2Templates(directory="app/templates")
|
||
NOMINATIM_USER_AGENT = os.getenv("NOMINATIM_USER_AGENT", "what2do-local/0.1")
|
||
SCHEDULER_INTERVAL = int(os.getenv("CRAWL_INTERVAL_MINUTES", "60"))
|
||
PICKER_SNAPSHOTS = {}
|
||
PICKER_LOCK = threading.Lock()
|
||
PICKER_TTL = timedelta(minutes=15)
|
||
EVENT_TAGS = {
|
||
"music": ("music", "band", "dj", "album", "tour", "concert", "punk", "emo", "rock", "singer"),
|
||
"art": ("art", "gallery", "exhibition", "paint", "creative", "workshop"),
|
||
"comedy": ("comedy", "comedian", "stand-up", "standup"),
|
||
"trivia": ("trivia", "quiz"),
|
||
"film": ("film", "movie", "cinema", "screening"),
|
||
"food": ("food", "dinner", "lunch", "brunch", "schnitty", "steak", "surf & turf"),
|
||
"sports": ("sport", "football", "rugby", "darts", "poker", "race"),
|
||
"community": ("community", "market", "fundraiser", "family", "festival"),
|
||
"nightlife": ("party", "club", "night", "bingo", "jackpot"),
|
||
}
|
||
|
||
SEEDS = [
|
||
{
|
||
"name": "Dicey Riley's weekly specials", "url": "https://www.diceyrileyshotel.com.au/pub-specials/",
|
||
"venue_name": "Dicey Riley's Hotel", "venue_address": "329-333 Crown St, Wollongong NSW 2500, Australia",
|
||
"latitude": -34.42538, "longitude": 150.89375,
|
||
"item_selector": "article.weekly_specials", "title_selector": "h2.elementor-heading-title",
|
||
"date_selector": ".elementor-post-info__item--type-custom", "time_selector": "", "link_selector": "a",
|
||
"date_format": "", "recurring_weekly": 1,
|
||
},
|
||
{
|
||
"name": "La La La's events", "url": "https://www.lalalas.com.au/all-events/",
|
||
"venue_name": "La La La's", "venue_address": "3 Globe Lane, Wollongong NSW 2500, Australia",
|
||
"latitude": -34.42569, "longitude": 150.89816,
|
||
"item_selector": ".slider_container > .items", "title_selector": ".feed_titles h2",
|
||
"date_selector": ".feed_dates ul", "time_selector": "", "link_selector": "a.slider_button",
|
||
"date_format": "%d %B %H:%M", "recurring_weekly": 0,
|
||
},
|
||
]
|
||
|
||
def source_values(form) -> dict:
|
||
raw_strategy = form.get("strategy_json", "{}").strip() or "{}"
|
||
try: strategy = json.loads(raw_strategy)
|
||
except json.JSONDecodeError as exc: raise ValueError(f"Strategy JSON is invalid: {exc.msg} at line {exc.lineno}") from exc
|
||
if strategy.get("mode", "html") == "html":
|
||
strategy.update({"mode": "html", "request_url": form.get("url", "").strip(), "method": strategy.get("method", "GET"), "response_type": "html", "item_selector": form.get("item_selector", "").strip(), "title_selector": form.get("title_selector", "").strip(), "date_selector": form.get("date_selector", "").strip(), "time_selector": form.get("time_selector", "").strip(), "link_selector": form.get("link_selector", "").strip(), "description_selector": form.get("description_selector", "").strip(), "location_selector": form.get("location_selector", "").strip(), "date_format": form.get("date_format", "").strip(), "recurring_weekly": 1 if form.get("recurring_weekly") else 0})
|
||
return {
|
||
"name": form.get("name", "").strip(), "url": form.get("url", "").strip(),
|
||
"venue_name": form.get("venue_name", "").strip(), "venue_address": form.get("venue_address", "").strip(),
|
||
"latitude": float(form["latitude"]) if form.get("latitude") else None,
|
||
"longitude": float(form["longitude"]) if form.get("longitude") else None,
|
||
"timezone": form.get("timezone", "Australia/Sydney").strip(), "item_selector": form.get("item_selector", "").strip(),
|
||
"title_selector": form.get("title_selector", "").strip(), "date_selector": form.get("date_selector", "").strip(),
|
||
"time_selector": form.get("time_selector", "").strip(), "link_selector": form.get("link_selector", "").strip(),
|
||
"description_selector": form.get("description_selector", "").strip(), "location_selector": form.get("location_selector", "").strip(),
|
||
"date_format": form.get("date_format", "").strip(), "recurring_weekly": 1 if form.get("recurring_weekly") else 0,
|
||
"strategy_json": json.dumps(strategy),
|
||
"enabled": 1 if form.get("enabled") else 0, "crawl_interval_minutes": int(form.get("crawl_interval_minutes") or 360),
|
||
}
|
||
|
||
def seed_sources():
|
||
with connect() as db:
|
||
for source in SEEDS:
|
||
source_strategy = {"mode": "html", "request_url": source["url"], "method": "GET", "response_type": "html", "item_selector": source["item_selector"], "title_selector": source["title_selector"], "date_selector": source["date_selector"], "time_selector": source.get("time_selector", ""), "link_selector": source.get("link_selector", ""), "link_attribute": "href", "description_selector": "", "location_selector": "", "date_format": source.get("date_format", ""), "recurring_weekly": source.get("recurring_weekly", 0)}
|
||
fields = {**source, "strategy_json": json.dumps(source_strategy), "timezone": "Australia/Sydney", "time_selector": source.get("time_selector", ""), "link_selector": source.get("link_selector", ""), "description_selector": "", "location_selector": "", "enabled": 1, "crawl_interval_minutes": 360}
|
||
columns = ",".join(fields)
|
||
db.execute(f"INSERT OR IGNORE INTO sources ({columns}) VALUES ({','.join('?' for _ in fields)})", tuple(fields.values()))
|
||
db.execute("UPDATE sources SET latitude=COALESCE(latitude,?),longitude=COALESCE(longitude,?) WHERE url=?", (source.get("latitude"), source.get("longitude"), source["url"]))
|
||
db.execute("UPDATE events SET latitude=COALESCE(latitude,?),longitude=COALESCE(longitude,?) WHERE source_id=(SELECT id FROM sources WHERE url=?)", (source.get("latitude"), source.get("longitude"), source["url"]))
|
||
|
||
def geocode(query: str):
|
||
if not query:
|
||
return None
|
||
try:
|
||
latitude, longitude = (float(value.strip()) for value in query.split(",", 1))
|
||
if -90 <= latitude <= 90 and -180 <= longitude <= 180:
|
||
return latitude, longitude, query
|
||
except (ValueError, TypeError):
|
||
pass
|
||
cache_key = query.casefold()
|
||
with connect() as db:
|
||
cached = db.execute("SELECT latitude, longitude, display_name FROM geocode_cache WHERE query=?", (cache_key,)).fetchone()
|
||
if cached:
|
||
return tuple(cached)
|
||
parts = [part.strip() for part in query.split(",") if part.strip()]
|
||
candidates = [query]
|
||
if len(parts) > 1:
|
||
candidates.append(", ".join(parts[1:]))
|
||
result = []
|
||
try:
|
||
for candidate in candidates:
|
||
response = httpx.get("https://nominatim.openstreetmap.org/search", params={"q": candidate, "format": "jsonv2", "limit": 1, "countrycodes": "au"}, headers={"User-Agent": NOMINATIM_USER_AGENT}, timeout=15)
|
||
response.raise_for_status()
|
||
result = response.json()
|
||
if result:
|
||
break
|
||
except (httpx.HTTPError, ValueError):
|
||
return None
|
||
if not result:
|
||
return None
|
||
value = (float(result[0]["lat"]), float(result[0]["lon"]), result[0].get("display_name", query))
|
||
with connect() as db:
|
||
db.execute("INSERT OR REPLACE INTO geocode_cache(query,latitude,longitude,display_name) VALUES(?,?,?,?)", (cache_key, *value))
|
||
return value
|
||
|
||
def crawl_source(source_id: int):
|
||
with connect() as db:
|
||
row = db.execute("SELECT * FROM sources WHERE id=?", (source_id,)).fetchone()
|
||
if not row:
|
||
raise CrawlError("Source not found.")
|
||
source = dict(row)
|
||
try:
|
||
with _http_client(timeout=45) as client:
|
||
events = execute_strategy(source, client)
|
||
coords = (source["latitude"], source["longitude"])
|
||
if None in coords:
|
||
point = geocode(source["venue_address"])
|
||
coords = point[:2] if point else (None, None)
|
||
seen = []
|
||
with connect() as db:
|
||
db.execute("UPDATE sources SET latitude=?,longitude=? WHERE id=?", (*coords, source_id))
|
||
for event in events:
|
||
key = external_key(event)
|
||
seen.append(key)
|
||
address = event.address or source["venue_address"]
|
||
db.execute("""INSERT INTO events(source_id,external_key,title,description,starts_at,ends_at,venue_name,address,latitude,longitude,event_url)
|
||
VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_key) DO UPDATE SET title=excluded.title,description=excluded.description,
|
||
starts_at=excluded.starts_at,ends_at=excluded.ends_at,venue_name=excluded.venue_name,address=excluded.address,latitude=excluded.latitude,longitude=excluded.longitude,event_url=excluded.event_url,last_seen_at=CURRENT_TIMESTAMP""",
|
||
(source_id, key, event.title, event.description, event.starts_at.astimezone(timezone.utc).isoformat(), event.ends_at.astimezone(timezone.utc).isoformat() if event.ends_at else None, source["venue_name"], address, *coords, event.url))
|
||
if seen:
|
||
placeholders = ",".join("?" for _ in seen)
|
||
db.execute(f"DELETE FROM events WHERE source_id=? AND external_key NOT IN ({placeholders})", (source_id, *seen))
|
||
else:
|
||
db.execute("DELETE FROM events WHERE source_id=?", (source_id,))
|
||
db.execute("UPDATE sources SET last_crawled_at=CURRENT_TIMESTAMP,last_status=?,last_error=NULL WHERE id=?", (f"Found {len(events)} events", source_id))
|
||
return len(events)
|
||
except Exception as exc:
|
||
with connect() as db:
|
||
db.execute("UPDATE sources SET last_crawled_at=CURRENT_TIMESTAMP,last_status='Failed',last_error=? WHERE id=?", (str(exc)[:1000], source_id))
|
||
raise
|
||
|
||
async def scheduler():
|
||
while True:
|
||
await asyncio.sleep(max(1, SCHEDULER_INTERVAL) * 60)
|
||
with connect() as db:
|
||
rows = db.execute("SELECT id,last_crawled_at,crawl_interval_minutes FROM sources WHERE enabled=1").fetchall()
|
||
now = datetime.now(timezone.utc)
|
||
for row in rows:
|
||
last = datetime.fromisoformat(row["last_crawled_at"] + "+00:00") if row["last_crawled_at"] else None
|
||
if not last or now - last >= timedelta(minutes=row["crawl_interval_minutes"]):
|
||
try: await asyncio.to_thread(crawl_source, row["id"])
|
||
except Exception: pass
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(app):
|
||
init_db(); seed_sources()
|
||
task = asyncio.create_task(scheduler())
|
||
yield
|
||
task.cancel()
|
||
|
||
app = FastAPI(title="What2Do", lifespan=lifespan)
|
||
app.mount("/static", StaticFiles(directory="app/static"), name="static")
|
||
|
||
def classify_event(title: str, description: str) -> list[str]:
|
||
text = f"{title} {description}".casefold()
|
||
tags = [tag for tag, keywords in EVENT_TAGS.items() if any(keyword in text for keyword in keywords)]
|
||
return tags or ["other"]
|
||
|
||
def distance_km(lat1, lon1, lat2, lon2):
|
||
p1, p2 = math.radians(lat1), math.radians(lat2)
|
||
dp, dl = math.radians(lat2-lat1), math.radians(lon2-lon1)
|
||
a = math.sin(dp/2)**2 + math.cos(p1)*math.cos(p2)*math.sin(dl/2)**2
|
||
return 6371 * 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
|
||
|
||
|
||
def event_overlaps_range(event_start: datetime, event_end: datetime | None, range_start: datetime, range_end: datetime) -> bool:
|
||
return event_start < range_end and (event_end or event_start) >= range_start
|
||
|
||
|
||
def calendar_day_counts(events: list[dict]) -> dict[str, int]:
|
||
counts = {}
|
||
for event in events:
|
||
covered_day = datetime.fromisoformat(event["day_key"]).date()
|
||
final_day = datetime.fromisoformat(event["end_day_key"]).date()
|
||
while covered_day <= final_day:
|
||
key = covered_day.isoformat()
|
||
counts[key] = counts.get(key, 0) + 1
|
||
covered_day += timedelta(days=1)
|
||
return counts
|
||
|
||
@app.get("/", response_class=HTMLResponse)
|
||
def index(request: Request, location: str = "", latitude: float | None = None, longitude: float | None = None, start: str = "", end: str = "", radius: float = 25, tag: list[str] = Query(default=[]), term: str = ""):
|
||
point = (latitude, longitude, location) if latitude is not None and longitude is not None else (geocode(location) if location else None)
|
||
local_now = datetime.now().astimezone()
|
||
default_start = local_now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||
if default_start.month == 12:
|
||
default_end = default_start.replace(year=default_start.year + 1, month=1)
|
||
else:
|
||
default_end = default_start.replace(month=default_start.month + 1)
|
||
if not start:
|
||
start = default_start.strftime("%Y-%m-%dT%H:%M")
|
||
if not end:
|
||
end = default_end.strftime("%Y-%m-%dT%H:%M")
|
||
try: start_at = datetime.fromisoformat(start).astimezone(timezone.utc)
|
||
except ValueError:
|
||
start_at = default_start.astimezone(timezone.utc)
|
||
start = default_start.strftime("%Y-%m-%dT%H:%M")
|
||
try: end_at = datetime.fromisoformat(end).astimezone(timezone.utc)
|
||
except ValueError:
|
||
end_at = default_end.astimezone(timezone.utc)
|
||
end = default_end.strftime("%Y-%m-%dT%H:%M")
|
||
with connect() as db:
|
||
rows = [dict(r) for r in db.execute("SELECT events.*,sources.name source_name FROM events JOIN sources ON sources.id=events.source_id WHERE events.hidden=0 AND starts_at < ? AND COALESCE(ends_at,starts_at) >= ? ORDER BY starts_at", (end_at.isoformat(), start_at.isoformat())).fetchall()]
|
||
for event in rows:
|
||
event["distance"] = distance_km(point[0], point[1], event["latitude"], event["longitude"]) if point and event["latitude"] is not None else None
|
||
local_start = datetime.fromisoformat(event["starts_at"]).astimezone()
|
||
event["display_time"] = local_start.strftime("%a %d %b %Y, %H:%M")
|
||
if event["ends_at"]:
|
||
local_end = datetime.fromisoformat(event["ends_at"]).astimezone()
|
||
event["display_time"] += f" – {local_end.strftime('%a %d %b %Y, %H:%M')}"
|
||
event["day_key"] = local_start.strftime("%Y-%m-%d")
|
||
event["end_day_key"] = datetime.fromisoformat(event["ends_at"]).astimezone().strftime("%Y-%m-%d") if event["ends_at"] else event["day_key"]
|
||
event["tags"] = classify_event(event["title"], event["description"])
|
||
if point: rows = [event for event in rows if event["distance"] is not None and event["distance"] <= radius]
|
||
selected_tags = [value for value in tag if value in EVENT_TAGS or value == "other"]
|
||
if selected_tags: rows = [event for event in rows if set(selected_tags).intersection(event["tags"])]
|
||
if term:
|
||
needle = term.casefold()
|
||
rows = [event for event in rows if needle in f"{event['title']} {event['description']} {event['venue_name']} {event['address']}".casefold()]
|
||
calendar_start = start_at.astimezone()
|
||
month_days = list(Calendar(firstweekday=0).itermonthdates(calendar_start.year, calendar_start.month))
|
||
day_counts = calendar_day_counts(rows)
|
||
calendar_days = [{"date": day, "key": day.strftime("%Y-%m-%d"), "count": day_counts.get(day.strftime("%Y-%m-%d"), 0), "in_month": day.month == calendar_start.month} for day in month_days]
|
||
map_events = [{"id": event["id"], "title": event["title"], "venue": event["venue_name"], "time": event["display_time"], "latitude": event["latitude"], "longitude": event["longitude"]} for event in rows if event["latitude"] is not None]
|
||
context = {"events": rows, "map_events": map_events, "calendar_days": calendar_days, "calendar_month": calendar_start.strftime("%B %Y"), "event_tags": EVENT_TAGS.keys(), "location": location, "latitude": latitude, "longitude": longitude, "start": start, "end": end, "radius": radius, "selected_tags": selected_tags, "term": term, "geocoded": point}
|
||
return TEMPLATES.TemplateResponse(request, "index.html", context)
|
||
|
||
@app.get("/api/locations")
|
||
def location_suggestions(q: str = ""):
|
||
query = q.strip()
|
||
if len(query) < 3:
|
||
return []
|
||
try:
|
||
response = httpx.get("https://photon.komoot.io/api/", params={"q": query, "limit": 10, "lang": "en"}, headers={"User-Agent": "what2do-local/0.1"}, timeout=10)
|
||
response.raise_for_status()
|
||
suggestions = []
|
||
for feature in response.json().get("features", []):
|
||
properties = feature.get("properties", {})
|
||
if properties.get("countrycode", "").upper() != "AU":
|
||
continue
|
||
coordinates = feature.get("geometry", {}).get("coordinates", [])
|
||
if len(coordinates) < 2:
|
||
continue
|
||
place = properties.get("name") or properties.get("city") or properties.get("district") or properties.get("county")
|
||
label = ", ".join(dict.fromkeys(value for value in (place, properties.get("city"), properties.get("state"), properties.get("postcode"), properties.get("country")) if value))
|
||
suggestion = {"label": label, "latitude": float(coordinates[1]), "longitude": float(coordinates[0])}
|
||
if label and suggestion not in suggestions:
|
||
suggestions.append(suggestion)
|
||
return suggestions[:5]
|
||
except (httpx.HTTPError, ValueError, KeyError, TypeError):
|
||
return []
|
||
|
||
@app.get("/api/addresses")
|
||
def address_suggestions(q: str = ""):
|
||
query = q.strip()
|
||
if len(query) < 3:
|
||
return []
|
||
try:
|
||
response = httpx.get("https://photon.komoot.io/api/", params={"q": query, "limit": 8, "lang": "en"}, headers={"User-Agent": "what2do-local/0.1"}, timeout=10)
|
||
response.raise_for_status()
|
||
results = []
|
||
for feature in response.json().get("features", []):
|
||
properties = feature.get("properties", {})
|
||
if properties.get("countrycode", "").upper() != "AU": continue
|
||
coordinates = feature.get("geometry", {}).get("coordinates", [])
|
||
if len(coordinates) < 2: continue
|
||
street = " ".join(value for value in (properties.get("housenumber"), properties.get("street")) if value)
|
||
label = ", ".join(dict.fromkeys(value for value in (properties.get("name"), street, properties.get("city"), properties.get("state"), properties.get("postcode"), properties.get("country")) if value))
|
||
item = {"label": label, "latitude": float(coordinates[1]), "longitude": float(coordinates[0])}
|
||
if label and item not in results: results.append(item)
|
||
return results[:5]
|
||
except (httpx.HTTPError, ValueError, KeyError, TypeError):
|
||
return []
|
||
|
||
@app.get("/events/{event_id}", response_class=HTMLResponse)
|
||
def event_detail(request: Request, event_id: int):
|
||
with connect() as db:
|
||
row = db.execute("SELECT events.*,sources.name source_name,sources.url source_url FROM events JOIN sources ON sources.id=events.source_id WHERE events.id=? AND events.hidden=0", (event_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404)
|
||
event = dict(row)
|
||
local_start = datetime.fromisoformat(event["starts_at"]).astimezone()
|
||
event["display_date"] = local_start.strftime("%A, %d %B %Y")
|
||
event["display_time"] = local_start.strftime("%H:%M")
|
||
event["display_end"] = None
|
||
if event["ends_at"]:
|
||
local_end = datetime.fromisoformat(event["ends_at"]).astimezone()
|
||
event["display_end"] = local_end.strftime("%A, %d %B %Y at %H:%M")
|
||
event["canonical_url"] = str(request.url)
|
||
return TEMPLATES.TemplateResponse(request, "event.html", {"event": event})
|
||
|
||
@app.get("/sources", response_class=HTMLResponse)
|
||
def sources(request: Request):
|
||
with connect() as db: rows = db.execute("SELECT *, (SELECT count(*) FROM events WHERE source_id=sources.id) event_count FROM sources ORDER BY name").fetchall()
|
||
return TEMPLATES.TemplateResponse(request, "sources.html", {"sources": rows})
|
||
|
||
@app.get("/sources/new", response_class=HTMLResponse)
|
||
def new_source(request: Request):
|
||
return TEMPLATES.TemplateResponse(request, "source_form.html", {"source": {"enabled": 1, "timezone": "Australia/Sydney", "crawl_interval_minutes": 360, "strategy_json": "{}"}, "action": "/sources", "preview": None})
|
||
|
||
@app.get("/sources/{source_id}", response_class=HTMLResponse)
|
||
def edit_source(request: Request, source_id: int):
|
||
with connect() as db: source = db.execute("SELECT * FROM sources WHERE id=?", (source_id,)).fetchone()
|
||
if not source: raise HTTPException(404)
|
||
return TEMPLATES.TemplateResponse(request, "source_form.html", {"source": {**dict(source), "strategy_json": json.dumps(parse_strategy(source["strategy_json"]), indent=2)}, "action": f"/sources/{source_id}", "preview": None})
|
||
|
||
@app.post("/sources")
|
||
async def create_source(request: Request):
|
||
form = await request.form(); values = source_values(form)
|
||
with connect() as db:
|
||
columns = ",".join(values); cursor = db.execute(f"INSERT INTO sources ({columns}) VALUES ({','.join('?' for _ in values)})", tuple(values.values()))
|
||
return RedirectResponse(f"/sources/{cursor.lastrowid}", 303)
|
||
|
||
@app.post("/sources/{source_id}")
|
||
async def update_source(request: Request, source_id: int):
|
||
form = await request.form(); values = source_values(form)
|
||
with connect() as db: db.execute(f"UPDATE sources SET {','.join(f'{key}=?' for key in values)} WHERE id=?", (*values.values(), source_id))
|
||
return RedirectResponse(f"/sources/{source_id}", 303)
|
||
|
||
@app.get("/sources/{source_id}/events", response_class=HTMLResponse)
|
||
def source_events(request: Request, source_id: int):
|
||
with connect() as db:
|
||
source = db.execute("SELECT * FROM sources WHERE id=?", (source_id,)).fetchone()
|
||
events = db.execute("SELECT * FROM events WHERE source_id=? ORDER BY starts_at", (source_id,)).fetchall()
|
||
if not source: raise HTTPException(404)
|
||
return TEMPLATES.TemplateResponse(request, "source_events.html", {"source": source, "events": events})
|
||
|
||
@app.get("/sources/{source_id}/events/{event_id}", response_class=HTMLResponse)
|
||
def edit_source_event(request: Request, source_id: int, event_id: int):
|
||
with connect() as db:
|
||
source = db.execute("SELECT * FROM sources WHERE id=?", (source_id,)).fetchone()
|
||
event = db.execute("SELECT * FROM events WHERE id=? AND source_id=?", (event_id, source_id)).fetchone()
|
||
if not source or not event: raise HTTPException(404)
|
||
return TEMPLATES.TemplateResponse(request, "event_form.html", {"source": source, "event": event})
|
||
|
||
@app.post("/sources/{source_id}/events/{event_id}")
|
||
async def update_source_event(request: Request, source_id: int, event_id: int):
|
||
form = await request.form()
|
||
values = (form.get("title", "").strip(), form.get("description", "").strip(), form.get("starts_at", "").strip(), form.get("ends_at", "").strip() or None, form.get("venue_name", "").strip(), form.get("address", "").strip(), float(form["latitude"]) if form.get("latitude") else None, float(form["longitude"]) if form.get("longitude") else None, form.get("event_url", "").strip(), event_id, source_id)
|
||
with connect() as db:
|
||
db.execute("UPDATE events SET title=?,description=?,starts_at=?,ends_at=?,venue_name=?,address=?,latitude=?,longitude=?,event_url=? WHERE id=? AND source_id=?", values)
|
||
return RedirectResponse(f"/sources/{source_id}/events", 303)
|
||
|
||
@app.post("/sources/{source_id}/events/{event_id}/visibility")
|
||
def toggle_event_visibility(source_id: int, event_id: int):
|
||
with connect() as db:
|
||
db.execute("UPDATE events SET hidden=CASE hidden WHEN 1 THEN 0 ELSE 1 END WHERE id=? AND source_id=?", (event_id, source_id))
|
||
return RedirectResponse(f"/sources/{source_id}/events", 303)
|
||
|
||
@app.post("/sources/{source_id}/crawl")
|
||
def run_crawl(source_id: int):
|
||
try: crawl_source(source_id)
|
||
except Exception: pass
|
||
return RedirectResponse("/sources", 303)
|
||
|
||
@app.post("/sources/{source_id}/delete")
|
||
def delete_source(source_id: int):
|
||
with connect() as db: db.execute("DELETE FROM sources WHERE id=?", (source_id,))
|
||
return RedirectResponse("/sources", 303)
|
||
|
||
def sanitize_picker_html(html: str, source_url: str) -> str:
|
||
soup = BeautifulSoup(html, "html.parser")
|
||
for tag in soup.select("script, iframe, frame, frameset, object, embed, form, input, button, textarea, select, base, meta[http-equiv]"):
|
||
tag.decompose()
|
||
for node in soup.find_all(True):
|
||
for attribute in list(node.attrs):
|
||
lowered = attribute.lower()
|
||
if lowered.startswith("on") or lowered in {"srcdoc", "integrity", "nonce", "formaction"}:
|
||
del node.attrs[attribute]
|
||
for attribute in ("src", "href", "poster", "action"):
|
||
value = node.get(attribute)
|
||
if not value: continue
|
||
value = str(value).strip()
|
||
if value.lower().startswith(("javascript:", "data:text/html", "vbscript:")):
|
||
del node.attrs[attribute]
|
||
elif attribute in {"src", "poster"}:
|
||
node.attrs[attribute] = urljoin(source_url, value)
|
||
elif attribute == "href" and node.name == "link" and "stylesheet" in (node.get("rel") or []):
|
||
node.attrs[attribute] = urljoin(source_url, value)
|
||
elif attribute in {"href", "action"}:
|
||
node.attrs[attribute] = "#"
|
||
node.attrs.pop("target", None)
|
||
head = soup.head
|
||
if not head:
|
||
head = soup.new_tag("head")
|
||
if soup.html: soup.html.insert(0, head)
|
||
else: soup.insert(0, head)
|
||
style = soup.new_tag("style")
|
||
style.string = "html,body{min-height:100%;background:white;color:#17211b} body{margin:12px} *{cursor:crosshair!important} img{max-width:100%;height:auto} .w2d-hover{outline:3px solid #dd5b32!important;outline-offset:2px!important}.w2d-picked{outline:4px solid #234c3c!important;outline-offset:2px!important}.w2d-container{outline:3px solid #dd5b32!important;outline-offset:2px!important;background-color:#dd5b3214!important}.w2d-field{outline:3px dashed #234c3c!important;outline-offset:1px!important;background-color:#234c3c1a!important}"
|
||
head.append(style)
|
||
script = soup.new_tag("script")
|
||
script.string = PICKER_SCRIPT
|
||
soup.body.append(script) if soup.body else soup.append(script)
|
||
return str(soup)
|
||
|
||
PICKER_SCRIPT = r'''(() => {
|
||
let hovered = null, picked = null, containerSelector = '', fieldSelectors = {};
|
||
const esc = value => CSS.escape(value);
|
||
function stableClasses(element) {
|
||
return [...element.classList].filter(name => !name.startsWith('w2d-') && !/^elementor-element-[a-f0-9]+$/.test(name) && !/^post-\d+$/.test(name)).slice(0, 3);
|
||
}
|
||
function idPattern(id) {
|
||
const match = id.match(/^(.*?[-_])\d{3,}$/);
|
||
return match && match[1].length >= 3 ? `[id^="${CSS.escape(match[1])}"]` : `#${esc(id)}`;
|
||
}
|
||
function simpleSelector(element) {
|
||
if (element.id) return idPattern(element.id);
|
||
const classes = stableClasses(element);
|
||
return element.tagName.toLowerCase() + classes.map(name => `.${esc(name)}`).join('');
|
||
}
|
||
function selectorFor(element) {
|
||
if (element.id) return idPattern(element.id);
|
||
const parts = [];
|
||
let node = element;
|
||
while (node && node.nodeType === 1 && node.tagName.toLowerCase() !== 'html') {
|
||
let part = node.tagName.toLowerCase();
|
||
const classes = stableClasses(node).slice(0, 2);
|
||
if (classes.length) part += classes.map(name => `.${esc(name)}`).join('');
|
||
const parent = node.parentElement;
|
||
if (parent && parent.querySelectorAll(`:scope > ${part}`).length > 1) {
|
||
const siblings = [...parent.children].filter(child => child.matches(part));
|
||
part += `:nth-of-type(${[...parent.children].indexOf(node) + 1})`;
|
||
}
|
||
parts.unshift(part);
|
||
const candidate = parts.join(' > ');
|
||
try { if (document.querySelectorAll(candidate).length === 1 || parts.length >= 4) return candidate; } catch (_) {}
|
||
node = parent;
|
||
}
|
||
return parts.join(' > ');
|
||
}
|
||
function relativeSelector(element, container) {
|
||
if (element === container) return ':scope';
|
||
const parts = [];
|
||
let node = element;
|
||
while (node && node !== container) {
|
||
let part = simpleSelector(node);
|
||
if (!node.id) {
|
||
const parent = node.parentElement;
|
||
if (parent) {
|
||
const matches = [...parent.children].filter(child => child.matches(part));
|
||
if (matches.length > 1) part += `:nth-of-type(${[...parent.children].indexOf(node) + 1})`;
|
||
}
|
||
}
|
||
parts.unshift(part); node = node.parentElement;
|
||
}
|
||
return parts.join(' > ');
|
||
}
|
||
function repeatedAncestor(element) {
|
||
let node = element;
|
||
let best = null;
|
||
let depth = 0;
|
||
while (node && node.tagName && node.tagName.toLowerCase() !== 'body') {
|
||
const selector = simpleSelector(node);
|
||
let count = 0;
|
||
try { count = document.querySelectorAll(selector).length; } catch (_) {}
|
||
if (count >= 2 && count <= 300) {
|
||
const text = node.textContent.trim();
|
||
const hasUsefulText = text.length >= 10;
|
||
const score = (hasUsefulText ? 30 : 0) + Math.min(count, 20) - depth * 4 - (node.querySelectorAll('*').length > 300 ? 20 : 0);
|
||
if (!best || score > best.score) best = { node, selector, count, score };
|
||
}
|
||
node = node.parentElement; depth += 1;
|
||
}
|
||
return best;
|
||
}
|
||
function safeSelect(root, selector) {
|
||
try { return selector === ':scope' ? [root] : [...root.querySelectorAll(selector)]; } catch (_) { return []; }
|
||
}
|
||
function clearHighlights(className) {
|
||
document.querySelectorAll(`.${className}`).forEach(node => node.classList.remove(className));
|
||
}
|
||
function containers() {
|
||
return containerSelector ? safeSelect(document, containerSelector) : [];
|
||
}
|
||
function preview() {
|
||
const nodes = containers();
|
||
const fields = {};
|
||
const inconsistencies = [];
|
||
Object.entries(fieldSelectors).forEach(([target, selector]) => {
|
||
const counts = nodes.map(node => safeSelect(node, selector).length);
|
||
const present = counts.filter(count => count === 1).length;
|
||
fields[target] = { selector, present, total: nodes.length, counts };
|
||
if (counts.some(count => count !== 1)) inconsistencies.push({ target, selector, present, total: nodes.length, multiple: counts.filter(count => count > 1).length });
|
||
});
|
||
const rows = nodes.slice(0, 20).map((node, index) => {
|
||
const values = { index: index + 1 };
|
||
Object.entries(fieldSelectors).forEach(([target, selector]) => {
|
||
const matches = safeSelect(node, selector);
|
||
values[target] = matches.length === 1 ? (matches[0].textContent.trim().replace(/\s+/g, ' ').slice(0, 160) || matches[0].getAttribute('href') || '') : '';
|
||
});
|
||
return values;
|
||
});
|
||
parent.postMessage({ type: 'what2do-picker-preview', containerSelector, containerCount: nodes.length, fields, inconsistencies, rows }, location.origin);
|
||
}
|
||
function highlight() {
|
||
clearHighlights('w2d-container'); clearHighlights('w2d-field');
|
||
containers().forEach(node => node.classList.add('w2d-container'));
|
||
containers().forEach(node => Object.values(fieldSelectors).forEach(selector => safeSelect(node, selector).forEach(match => match.classList.add('w2d-field'))));
|
||
}
|
||
window.addEventListener('message', event => {
|
||
if (event.source !== parent || event.data?.type !== 'what2do-picker-config') return;
|
||
containerSelector = event.data.containerSelector || '';
|
||
fieldSelectors = event.data.fieldSelectors || {};
|
||
highlight(); preview();
|
||
});
|
||
document.addEventListener('mouseover', event => {
|
||
event.preventDefault(); event.stopPropagation();
|
||
if (hovered) hovered.classList.remove('w2d-hover');
|
||
hovered = event.target; hovered.classList.add('w2d-hover');
|
||
}, true);
|
||
document.addEventListener('mouseout', event => { if (hovered) hovered.classList.remove('w2d-hover'); }, true);
|
||
document.addEventListener('click', event => {
|
||
event.preventDefault(); event.stopPropagation();
|
||
if (picked) picked.classList.remove('w2d-picked');
|
||
picked = event.target; picked.classList.add('w2d-picked');
|
||
const selector = selectorFor(picked);
|
||
const repeated = repeatedAncestor(picked);
|
||
const selectedContainer = containerSelector ? picked.closest(containerSelector) : null;
|
||
const relative = selectedContainer ? relativeSelector(picked, selectedContainer) : (repeated ? relativeSelector(picked, repeated.node) : selector);
|
||
parent.postMessage({ type: 'what2do-selector', selector, selectorCount: document.querySelectorAll(selector).length, generalizedSelector: simpleSelector(picked), generalizedCount: document.querySelectorAll(simpleSelector(picked)).length, containerSelector: repeated?.selector || '', containerCount: repeated?.count || 0, withinConfiguredContainer: Boolean(selectedContainer), relativeSelector: relative, tag: picked.tagName.toLowerCase(), text: picked.textContent.trim().slice(0, 160), href: picked.getAttribute('href') }, location.origin);
|
||
}, true);
|
||
})();'''
|
||
|
||
@app.post("/picker/snapshots")
|
||
async def create_picker_snapshot(request: Request):
|
||
form = await request.form()
|
||
url = form.get("url", "").strip()
|
||
try:
|
||
sanitized = sanitize_picker_html(fetch_html(url), url)
|
||
except Exception as exc:
|
||
return JSONResponse({"error": str(exc)}, status_code=400)
|
||
token = secrets.token_urlsafe(24)
|
||
now = datetime.now(timezone.utc)
|
||
with PICKER_LOCK:
|
||
expired = [key for key, value in PICKER_SNAPSHOTS.items() if now - value["created_at"] > PICKER_TTL]
|
||
for key in expired: PICKER_SNAPSHOTS.pop(key, None)
|
||
PICKER_SNAPSHOTS[token] = {"html": sanitized, "created_at": now}
|
||
return {"preview_url": f"/picker/snapshots/{token}"}
|
||
|
||
@app.get("/picker/snapshots/{token}", response_class=HTMLResponse)
|
||
def picker_snapshot(token: str):
|
||
with PICKER_LOCK:
|
||
snapshot = PICKER_SNAPSHOTS.get(token)
|
||
if not snapshot or datetime.now(timezone.utc) - snapshot["created_at"] > PICKER_TTL:
|
||
raise HTTPException(404, "Picker snapshot expired")
|
||
return HTMLResponse(snapshot["html"], headers={"Content-Security-Policy": "default-src 'none'; img-src https: http: data:; style-src 'unsafe-inline' https:; font-src https: data:; script-src 'unsafe-inline'; connect-src 'none'; frame-src 'none'; form-action 'none'; base-uri 'none'", "Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"})
|
||
|
||
@app.post("/auto-detect", response_class=HTMLResponse)
|
||
async def auto_detect(request: Request):
|
||
form = await request.form()
|
||
url = form.get("url", "").strip()
|
||
source = {
|
||
"id": form.get("source_id"), "name": form.get("name", ""), "url": url,
|
||
"venue_name": form.get("venue_name", ""), "venue_address": form.get("venue_address", ""),
|
||
"latitude": form.get("latitude") or None, "longitude": form.get("longitude") or None,
|
||
"timezone": form.get("timezone", "Australia/Sydney"), "enabled": 1,
|
||
"crawl_interval_minutes": form.get("crawl_interval_minutes", 360),
|
||
}
|
||
try:
|
||
html = fetch_html(url)
|
||
rules, items, message = detect_rules(html, url, source["timezone"])
|
||
strategy = {"mode": "html", "request_url": url, "method": "GET", "response_type": "html", **rules, "link_attribute": "href"}
|
||
source["strategy_json"] = json.dumps(strategy, indent=2)
|
||
with _http_client(timeout=45) as client:
|
||
strategy_items = execute_strategy({**source, "strategy_json": source["strategy_json"]}, client)
|
||
if strategy_items:
|
||
items = strategy_items
|
||
inferred = infer_page_fields(html, url)
|
||
for key, value in inferred.items():
|
||
if value and not source.get(key): source[key] = value
|
||
source.update(rules)
|
||
if source.get("venue_address") and not source.get("latitude"):
|
||
point = geocode(source["venue_address"])
|
||
if point: source["latitude"], source["longitude"] = point[:2]
|
||
preview_data = [{"title": item.title, "starts_at": item.starts_at.isoformat(), "ends_at": item.ends_at.isoformat() if item.ends_at else None, "url": item.url} for item in items[:20]]
|
||
error = None
|
||
except Exception as exc:
|
||
source.update({"item_selector": "", "title_selector": "", "date_selector": "", "time_selector": "", "link_selector": "", "description_selector": "", "location_selector": "", "date_format": "", "recurring_weekly": 0, "strategy_json": "{}"})
|
||
preview_data, message, error = [], None, str(exc)
|
||
action = f"/sources/{source['id']}" if source.get("id") else "/sources"
|
||
return TEMPLATES.TemplateResponse(request, "source_form.html", {"source": source, "action": action, "preview": preview_data, "preview_error": error, "detect_message": message})
|
||
|
||
@app.post("/preview", response_class=HTMLResponse)
|
||
async def preview(request: Request):
|
||
form = await request.form(); values = source_values(form); values["url"] = form.get("url", "")
|
||
try:
|
||
with _http_client(timeout=45) as client:
|
||
items = execute_strategy(values, client)[:20]
|
||
preview_data = [{"title": item.title, "starts_at": item.starts_at.isoformat(), "ends_at": item.ends_at.isoformat() if item.ends_at else None, "url": item.url} for item in items]
|
||
error = None
|
||
except Exception as exc: preview_data, error = [], str(exc)
|
||
source_id = form.get("source_id")
|
||
return TEMPLATES.TemplateResponse(request, "source_form.html", {"source": values, "action": f"/sources/{source_id}" if source_id else "/sources", "preview": preview_data, "preview_error": error})
|