diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d5624cb --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.coverage +htmlcov/ +test-results/*.txt +.venv/ +data/ +*.db diff --git a/Dockerfile b/Dockerfile index dfa7562..31ab55b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,9 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY app ./app -COPY README.md . +COPY tests ./tests +COPY pytest.ini . +COPY README.md STRATEGIES.md ./ RUN mkdir -p /data EXPOSE 8000 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index bb6775b..7e2dadd 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,22 @@ Use **Test extraction** to preview up to 20 records before saving. Static server Addresses are geocoded and cached using OpenStreetMap's public Nominatim service. The search field also accepts `latitude,longitude` directly, and every source has optional coordinate fields for deterministic/offline venue positioning. Set a real contact in `NOMINATIM_USER_AGENT` before public/shared use and follow the [Nominatim usage policy](https://operations.osmfoundation.org/policies/nominatim/). For heavier use, configure or self-host a compatible geocoder. +## Tests + +The approved 50-source Illawarra corpus is stored in `tests/fixtures/illawarra_sources.json`. Normal tests are fixture-based and do not access live websites: + +```sh +docker compose --profile test run --rm test +``` + +Detailed per-test outcomes, durations, failure traces, and totals are exported to `test-results/latest.txt` after every run. The file is replaced on the next run and is ignored by Git. + +Run optional live reachability checks separately (results depend on third-party availability and access policies): + +```sh +docker compose --profile test run --rm test pytest -m network tests/integration +``` + ## Operational notes - SQLite data is persisted in the `what2do-data` Docker volume. diff --git a/app/crawler.py b/app/crawler.py index c02aa43..1b96bef 100644 --- a/app/crawler.py +++ b/app/crawler.py @@ -105,15 +105,17 @@ def _parse_date(value: str, source: dict, now: datetime) -> datetime: candidates = [fmt] # Event lists commonly omit a year. Resolve to the next occurrence. for candidate in candidates: - parsed = datetime.strptime(value, candidate).replace(tzinfo=tz) - if "%Y" not in candidate and "%y" not in candidate: - parsed = parsed.replace(year=now.year) + has_year = "%Y" in candidate or "%y" in candidate + parse_value = value if has_year else f"{value} {now.year}" + parse_format = candidate if has_year else f"{candidate} %Y" + parsed = datetime.strptime(parse_value, parse_format).replace(tzinfo=tz) + if not has_year: if parsed < now - timedelta(days=2): parsed = parsed.replace(year=now.year + 1) return parsed if not re.search(r"\b\d{1,4}\b", value): raise ValueError("Date text must contain a numeric day or year") - parsed = date_parser.parse(value, fuzzy=True, default=now.replace(tzinfo=None), tzinfos={"INFO": None, "IPAC": None, "IMAX": None}) + parsed = date_parser.parse(value, fuzzy=True, default=now.replace(tzinfo=None, second=0, microsecond=0), tzinfos={"INFO": None, "IPAC": None, "IMAX": None}) if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=tz) if not now.year - 1 <= parsed.year <= now.year + 10: diff --git a/app/main.py b/app/main.py index 710d281..58d3565 100644 --- a/app/main.py +++ b/app/main.py @@ -149,6 +149,8 @@ def crawl_source(source_id: int): if seen: placeholders = ",".join("?" for _ in seen) db.execute(f"DELETE FROM events WHERE source_id=? AND external_key NOT IN ({placeholders})", (source_id, *seen)) + else: + db.execute("DELETE FROM events WHERE source_id=?", (source_id,)) db.execute("UPDATE sources SET last_crawled_at=CURRENT_TIMESTAMP,last_status=?,last_error=NULL WHERE id=?", (f"Found {len(events)} events", source_id)) return len(events) except Exception as exc: @@ -189,6 +191,22 @@ def distance_km(lat1, lon1, lat2, lon2): a = math.sin(dp/2)**2 + math.cos(p1)*math.cos(p2)*math.sin(dl/2)**2 return 6371 * 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) + +def event_overlaps_range(event_start: datetime, event_end: datetime | None, range_start: datetime, range_end: datetime) -> bool: + return event_start < range_end and (event_end or event_start) >= range_start + + +def calendar_day_counts(events: list[dict]) -> dict[str, int]: + counts = {} + for event in events: + covered_day = datetime.fromisoformat(event["day_key"]).date() + final_day = datetime.fromisoformat(event["end_day_key"]).date() + while covered_day <= final_day: + key = covered_day.isoformat() + counts[key] = counts.get(key, 0) + 1 + covered_day += timedelta(days=1) + return counts + @app.get("/", response_class=HTMLResponse) def index(request: Request, location: str = "", latitude: float | None = None, longitude: float | None = None, start: str = "", end: str = "", radius: float = 25, tag: list[str] = Query(default=[]), term: str = ""): point = (latitude, longitude, location) if latitude is not None and longitude is not None else (geocode(location) if location else None) @@ -230,14 +248,7 @@ def index(request: Request, location: str = "", latitude: float | None = None, l rows = [event for event in rows if needle in f"{event['title']} {event['description']} {event['venue_name']} {event['address']}".casefold()] calendar_start = start_at.astimezone() month_days = list(Calendar(firstweekday=0).itermonthdates(calendar_start.year, calendar_start.month)) - day_counts = {} - for event in rows: - covered_day = datetime.fromisoformat(event["day_key"]).date() - final_day = datetime.fromisoformat(event["end_day_key"]).date() - while covered_day <= final_day: - key = covered_day.strftime("%Y-%m-%d") - day_counts[key] = day_counts.get(key, 0) + 1 - covered_day += timedelta(days=1) + day_counts = calendar_day_counts(rows) calendar_days = [{"date": day, "key": day.strftime("%Y-%m-%d"), "count": day_counts.get(day.strftime("%Y-%m-%d"), 0), "in_month": day.month == calendar_start.month} for day in month_days] map_events = [{"id": event["id"], "title": event["title"], "venue": event["venue_name"], "time": event["display_time"], "latitude": event["latitude"], "longitude": event["longitude"]} for event in rows if event["latitude"] is not None] context = {"events": rows, "map_events": map_events, "calendar_days": calendar_days, "calendar_month": calendar_start.strftime("%B %Y"), "event_tags": EVENT_TAGS.keys(), "location": location, "latitude": latitude, "longitude": longitude, "start": start, "end": end, "radius": radius, "selected_tags": selected_tags, "term": term, "geocoded": point} @@ -404,7 +415,7 @@ def sanitize_picker_html(html: str, source_url: str) -> str: if soup.html: soup.html.insert(0, head) else: soup.insert(0, head) style = soup.new_tag("style") - style.string = "html,body{min-height:100%;background:white;color:#17211b} body{margin:12px} *{cursor:crosshair!important} img{max-width:100%;height:auto} .w2d-hover{outline:3px solid #dd5b32!important;outline-offset:2px!important}.w2d-picked{outline:4px solid #234c3c!important;outline-offset:2px!important}" + style.string = "html,body{min-height:100%;background:white;color:#17211b} body{margin:12px} *{cursor:crosshair!important} img{max-width:100%;height:auto} .w2d-hover{outline:3px solid #dd5b32!important;outline-offset:2px!important}.w2d-picked{outline:4px solid #234c3c!important;outline-offset:2px!important}.w2d-container{outline:3px solid #dd5b32!important;outline-offset:2px!important;background-color:#dd5b3214!important}.w2d-field{outline:3px dashed #234c3c!important;outline-offset:1px!important;background-color:#234c3c1a!important}" head.append(style) script = soup.new_tag("script") script.string = PICKER_SCRIPT @@ -412,7 +423,7 @@ def sanitize_picker_html(html: str, source_url: str) -> str: return str(soup) PICKER_SCRIPT = r'''(() => { - let hovered = null, picked = null; + let hovered = null, picked = null, containerSelector = '', fieldSelectors = {}; const esc = value => CSS.escape(value); function stableClasses(element) { return [...element.classList].filter(name => !name.startsWith('w2d-') && !/^elementor-element-[a-f0-9]+$/.test(name) && !/^post-\d+$/.test(name)).slice(0, 3); @@ -481,6 +492,46 @@ PICKER_SCRIPT = r'''(() => { } return best; } + function safeSelect(root, selector) { + try { return selector === ':scope' ? [root] : [...root.querySelectorAll(selector)]; } catch (_) { return []; } + } + function clearHighlights(className) { + document.querySelectorAll(`.${className}`).forEach(node => node.classList.remove(className)); + } + function containers() { + return containerSelector ? safeSelect(document, containerSelector) : []; + } + function preview() { + const nodes = containers(); + const fields = {}; + const inconsistencies = []; + Object.entries(fieldSelectors).forEach(([target, selector]) => { + const counts = nodes.map(node => safeSelect(node, selector).length); + const present = counts.filter(count => count === 1).length; + fields[target] = { selector, present, total: nodes.length, counts }; + if (counts.some(count => count !== 1)) inconsistencies.push({ target, selector, present, total: nodes.length, multiple: counts.filter(count => count > 1).length }); + }); + const rows = nodes.slice(0, 20).map((node, index) => { + const values = { index: index + 1 }; + Object.entries(fieldSelectors).forEach(([target, selector]) => { + const matches = safeSelect(node, selector); + values[target] = matches.length === 1 ? (matches[0].textContent.trim().replace(/\s+/g, ' ').slice(0, 160) || matches[0].getAttribute('href') || '') : ''; + }); + return values; + }); + parent.postMessage({ type: 'what2do-picker-preview', containerSelector, containerCount: nodes.length, fields, inconsistencies, rows }, location.origin); + } + function highlight() { + clearHighlights('w2d-container'); clearHighlights('w2d-field'); + containers().forEach(node => node.classList.add('w2d-container')); + containers().forEach(node => Object.values(fieldSelectors).forEach(selector => safeSelect(node, selector).forEach(match => match.classList.add('w2d-field')))); + } + window.addEventListener('message', event => { + if (event.source !== parent || event.data?.type !== 'what2do-picker-config') return; + containerSelector = event.data.containerSelector || ''; + fieldSelectors = event.data.fieldSelectors || {}; + highlight(); preview(); + }); document.addEventListener('mouseover', event => { event.preventDefault(); event.stopPropagation(); if (hovered) hovered.classList.remove('w2d-hover'); @@ -493,7 +544,9 @@ PICKER_SCRIPT = r'''(() => { picked = event.target; picked.classList.add('w2d-picked'); const selector = selectorFor(picked); const repeated = repeatedAncestor(picked); - parent.postMessage({ type: 'what2do-selector', selector, selectorCount: document.querySelectorAll(selector).length, generalizedSelector: simpleSelector(picked), generalizedCount: document.querySelectorAll(simpleSelector(picked)).length, containerSelector: repeated?.selector || '', containerCount: repeated?.count || 0, relativeSelector: repeated ? relativeSelector(picked, repeated.node) : selector, tag: picked.tagName.toLowerCase(), text: picked.textContent.trim().slice(0, 160), href: picked.getAttribute('href') }, location.origin); + const selectedContainer = containerSelector ? picked.closest(containerSelector) : null; + const relative = selectedContainer ? relativeSelector(picked, selectedContainer) : (repeated ? relativeSelector(picked, repeated.node) : selector); + parent.postMessage({ type: 'what2do-selector', selector, selectorCount: document.querySelectorAll(selector).length, generalizedSelector: simpleSelector(picked), generalizedCount: document.querySelectorAll(simpleSelector(picked)).length, containerSelector: repeated?.selector || '', containerCount: repeated?.count || 0, withinConfiguredContainer: Boolean(selectedContainer), relativeSelector: relative, tag: picked.tagName.toLowerCase(), text: picked.textContent.trim().slice(0, 160), href: picked.getAttribute('href') }, location.origin); }, true); })();''' diff --git a/app/static/source.js b/app/static/source.js index 613f70e..da29cf4 100644 --- a/app/static/source.js +++ b/app/static/source.js @@ -7,10 +7,37 @@ const pickedText = document.querySelector('#picked-text'); const pickedCount = document.querySelector('#picked-count'); const pickedContainer = document.querySelector('#picked-container'); const pickedRelative = document.querySelector('#picked-relative'); -const pickerTarget = document.querySelector('#picker-target'); const applySelector = document.querySelector('#apply-selector'); -const applyPattern = document.querySelector('#apply-pattern'); +const resetPicker = document.querySelector('#reset-picker'); +const pickerSteps = document.querySelector('#picker-steps'); +const pickerWorkspace = document.querySelector('#picker-workspace'); +const pickerContainerCount = document.querySelector('#picker-container-count'); +const pickerWarning = document.querySelector('#picker-warning'); +const pickerPreviewEmpty = document.querySelector('#picker-preview-empty'); +const pickerPreviewRows = document.querySelector('#picker-preview-rows'); +const guidedTargets = ['item_selector', 'title_selector', 'date_selector', 'time_selector']; +const targetLabels = { item_selector: 'event container', title_selector: 'title', date_selector: 'date or range', time_selector: 'time' }; let currentSelection = null; +let pickerStep = 0; +let containerSelector = ''; +let fieldSelectors = {}; + +function currentTarget() { return guidedTargets[Math.min(pickerStep, guidedTargets.length - 1)]; } +function renderPickerSteps() { + pickerSteps?.querySelectorAll('li').forEach((step, index) => { + step.classList.toggle('active', index === pickerStep); + step.classList.toggle('complete', index < pickerStep); + }); +} +function sendPickerConfig() { + pickerFrame?.contentWindow?.postMessage({ type: 'what2do-picker-config', containerSelector, fieldSelectors }, location.origin); +} +function resetPickerState() { + currentSelection = null; pickerStep = 0; containerSelector = ''; fieldSelectors = {}; + pickerSelection.hidden = true; pickerWarning.hidden = true; pickerPreviewRows.replaceChildren(); pickerPreviewEmpty.hidden = false; + pickerContainerCount.textContent = ''; pickerStatus.textContent = 'Step 1: click one complete event container.'; + renderPickerSteps(); sendPickerConfig(); +} if (loadPicker && pickerFrame) loadPicker.addEventListener('click', async () => { const url = document.querySelector('input[name="url"]')?.value.trim(); @@ -21,22 +48,31 @@ if (loadPicker && pickerFrame) loadPicker.addEventListener('click', async () => const response = await fetch('/picker/snapshots', { method: 'POST', body }); const result = await response.json(); if (!response.ok) throw new Error(result.error || 'Unable to create preview'); - pickerFrame.src = result.preview_url; pickerFrame.hidden = false; - pickerStatus.textContent = 'Hover to highlight, then click an element.'; + pickerFrame.src = result.preview_url; pickerWorkspace.hidden = false; pickerSteps.hidden = false; resetPicker.hidden = false; + resetPickerState(); } catch (error) { pickerStatus.textContent = error.message; } finally { loadPicker.disabled = false; } }); +if (resetPicker) resetPicker.addEventListener('click', resetPickerState); +if (pickerFrame) pickerFrame.addEventListener('load', sendPickerConfig); window.addEventListener('message', event => { - if (!pickerFrame || event.source !== pickerFrame.contentWindow || event.data?.type !== 'what2do-selector') return; - currentSelection = event.data; - pickedSelector.textContent = event.data.selector; - pickedCount.textContent = `${event.data.selectorCount} exact match${event.data.selectorCount === 1 ? '' : 'es'}; ${event.data.generalizedCount} similar elements`; - pickedContainer.textContent = event.data.containerSelector || 'No repeated ancestor detected'; - pickedRelative.textContent = event.data.containerSelector ? `${event.data.containerCount} containers · child: ${event.data.relativeSelector}` : ''; - pickedText.textContent = event.data.text || event.data.tag; - applyPattern.disabled = !event.data.containerSelector; - pickerSelection.hidden = false; + if (!pickerFrame || event.source !== pickerFrame.contentWindow) return; + if (event.data?.type === 'what2do-selector') { + if (pickerStep >= guidedTargets.length) return; + currentSelection = event.data; + const target = currentTarget(); + pickedSelector.textContent = target === 'item_selector' ? (event.data.containerSelector || event.data.generalizedSelector) : event.data.relativeSelector; + pickedCount.textContent = target === 'item_selector' ? `${event.data.containerCount || event.data.generalizedCount} matching containers` : 'Corresponding field selector'; + pickedContainer.textContent = target === 'item_selector' ? (event.data.containerSelector || event.data.generalizedSelector) : containerSelector; + pickedRelative.textContent = target === 'item_selector' ? 'All matches will be highlighted.' : event.data.relativeSelector; + pickedText.textContent = event.data.text || event.data.tag; + applySelector.disabled = target !== 'item_selector' && !event.data.withinConfiguredContainer; + applySelector.textContent = `Use as ${targetLabels[target]}`; + pickerSelection.hidden = false; + pickerStatus.textContent = applySelector.disabled ? 'Pick a field inside one of the highlighted event containers.' : `Review this ${targetLabels[target]}, then confirm.`; + } + if (event.data?.type === 'what2do-picker-preview') renderExtractionPreview(event.data); }); function updateSelectorField(target, selector) { @@ -51,18 +87,39 @@ function updateSelectorField(target, selector) { } catch (_) {} } +function renderExtractionPreview(data) { + pickerContainerCount.textContent = data.containerCount ? `${data.containerCount} containers` : ''; + pickerPreviewRows.replaceChildren(); + data.rows.forEach(row => { + const item = document.createElement('div'); + item.className = 'picker-preview-row'; + const title = document.createElement('strong'); title.textContent = row.title_selector || `Container ${row.index}`; item.append(title); + const details = [row.date_selector, row.time_selector].filter(Boolean).join(' · '); + if (details) { const meta = document.createElement('span'); meta.textContent = details; item.append(meta); } + pickerPreviewRows.append(item); + }); + pickerPreviewEmpty.hidden = data.rows.length > 0; + if (data.inconsistencies.length) { + pickerWarning.textContent = data.inconsistencies.map(issue => `${targetLabels[issue.target] || issue.target}: exactly one match in ${issue.present}/${issue.total} containers${issue.multiple ? `; multiple matches in ${issue.multiple}` : ''}`).join(' '); + pickerWarning.hidden = false; + } else pickerWarning.hidden = true; +} + if (applySelector) applySelector.addEventListener('click', () => { if (!currentSelection) return; - updateSelectorField(pickerTarget.value, currentSelection.selector); - pickerStatus.textContent = `Applied exact selector to ${pickerTarget.options[pickerTarget.selectedIndex].text}.`; -}); - -if (applyPattern) applyPattern.addEventListener('click', () => { - if (!currentSelection?.containerSelector) return; - const target = pickerTarget.value; - updateSelectorField('item_selector', currentSelection.containerSelector); - updateSelectorField(target, target === 'item_selector' ? currentSelection.containerSelector : currentSelection.relativeSelector); - pickerStatus.textContent = `Matched ${currentSelection.containerCount} event containers and applied a relative ${pickerTarget.options[pickerTarget.selectedIndex].text.toLowerCase()} selector.`; + const target = currentTarget(); + if (target === 'item_selector') { + containerSelector = currentSelection.containerSelector || currentSelection.generalizedSelector; + if (!containerSelector) return; + updateSelectorField(target, containerSelector); + } else { + if (!currentSelection.withinConfiguredContainer) return; + fieldSelectors[target] = currentSelection.relativeSelector; + updateSelectorField(target, currentSelection.relativeSelector); + } + pickerStep += 1; currentSelection = null; pickerSelection.hidden = true; sendPickerConfig(); renderPickerSteps(); + if (pickerStep < guidedTargets.length) pickerStatus.textContent = `Step ${pickerStep + 1}: pick the ${targetLabels[currentTarget()]} inside a highlighted container.`; + else pickerStatus.textContent = 'Guided setup complete. Review the preview and any consistency warnings.'; }); const strategyTemplates = { diff --git a/app/static/style.css b/app/static/style.css index 853741b..c051759 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -1 +1 @@ -:root{--ink:#17211b;--muted:#657068;--paper:#f4f0e7;--card:#fffdf7;--accent:#dd5b32;--green:#234c3c;--line:#d9d4c8}*{box-sizing:border-box}body{margin:0;background:var(--paper);color:var(--ink);font:16px/1.5 system-ui,-apple-system,sans-serif}a{color:var(--green)}header{height:68px;padding:0 max(24px,calc((100% - 1120px)/2));display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid var(--line)}nav{display:flex;gap:24px}nav a{text-decoration:none;font-weight:650}.brand{font:bold 24px Georgia,serif;text-decoration:none;color:var(--ink)}main{max-width:1120px;margin:auto;padding:48px 24px 90px}h1,h2,h3{font-family:Georgia,serif;line-height:1.1;margin:.25em 0}h1{font-size:clamp(2.5rem,6vw,5.5rem);max-width:850px}h2{font-size:2rem}.hero{padding:45px 0}.hero p:not(.eyebrow){font-size:1.2rem;color:var(--muted)}.eyebrow{font-size:.75rem;font-weight:800;letter-spacing:.16em;color:var(--accent)}.card{background:var(--card);border:1px solid var(--line);border-radius:12px;box-shadow:0 5px 18px #17211b0a}.search{display:grid;grid-template-columns:2fr 1.4fr 1.4fr .8fr auto;gap:12px;padding:18px;align-items:end}label{display:flex;flex-direction:column;gap:6px;font-size:.8rem;font-weight:750;position:relative}input,select{width:100%;border:1px solid var(--line);background:white;border-radius:7px;padding:12px;color:var(--ink);font:inherit}.suffix{position:absolute;right:12px;bottom:12px;color:var(--muted)}button,.button{border:0;border-radius:7px;padding:12px 18px;background:var(--green);color:white;text-decoration:none;font-weight:750;font:inherit;cursor:pointer;display:inline-block}.secondary{background:transparent;color:var(--green);border:1px solid var(--green)}.danger{background:#9d302c}.section-head{display:flex;justify-content:space-between;align-items:end;margin:52px 0 22px;gap:20px}.section-head h1{font-size:3.5rem}.section-head p{max-width:650px;color:var(--muted)}.events,.source-grid{display:grid;gap:14px}.event{padding:24px}.event h3{font-size:1.55rem}.event p{color:var(--muted)}.date{color:var(--accent);font-weight:800}.event-foot,.source-title,.actions{display:flex;align-items:center;justify-content:space-between;gap:12px;margin-top:22px}.pill{font-size:.75rem;background:#eee9de;padding:5px 9px;border-radius:99px}.good{background:#dce9df;color:#285039}.empty{text-align:center;padding:60px}.source{padding:24px}.source-stats{display:flex;gap:30px;flex-wrap:wrap;border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:15px 0;margin-top:18px}.source-stats>div{display:grid}.source-stats small{color:var(--muted)}.admin-events{display:grid;gap:12px}.admin-event{padding:22px;display:flex;align-items:center;justify-content:space-between;gap:20px}.admin-event.is-hidden{opacity:.6;border-style:dashed}.event-admin-form{padding:25px;display:grid;gap:18px}textarea{width:100%;border:1px solid var(--line);background:white;border-radius:7px;padding:12px;color:var(--ink);font:inherit;resize:vertical}.source-title{margin-top:0}.source-title a{word-break:break-all;font-size:.85rem}dl{display:flex;gap:40px;border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:15px 0}dt{color:var(--muted);font-size:.75rem}dd{margin:2px 0;font-weight:700}.actions{justify-content:flex-start}.actions form{margin:0}.notice{padding:12px 16px;border-radius:7px}.error{background:#f4deda;color:#792a27}.success{background:#dce9df;color:#285039}.url-detect-row{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:8px}.url-detect-row .detect-submit{width:auto;white-space:nowrap;background:var(--green);color:white;border:0;font-weight:700}.address-field{position:relative}.picker-panel{grid-column:1/-1;padding:25px;display:grid;gap:14px}.picker-controls{display:flex;align-items:end;gap:10px}.picker-controls .field{min-width:220px}.selector-picker{width:100%;height:620px;border:1px solid var(--line);border-radius:8px;background:white}.picker-selection{display:grid;grid-template-columns:minmax(0,1fr) minmax(0,1fr);align-items:start;gap:14px;padding:14px;background:#eee9de;border-radius:8px}.picker-selection>div{display:grid;gap:5px}.picker-selection code{overflow-wrap:anywhere}.picker-selection span{color:var(--muted);font-size:.8rem;overflow:hidden;text-overflow:ellipsis}.picker-actions{display:flex!important;flex-direction:row;flex-wrap:wrap}.picker-actions button{padding:9px 12px}.rules-details{grid-column:1/-1;overflow:hidden}.rules-details>summary{cursor:pointer;padding:22px;font:700 1.5rem Georgia,serif;display:flex;justify-content:space-between;gap:15px}.rules-details>summary span{font:400 .8rem system-ui;color:var(--muted)}.rules-details[open]>summary{border-bottom:1px solid var(--line)}.sticky-actions input[type=submit]{width:auto;background:var(--green);color:white;border:0;font-weight:700;cursor:pointer}.sticky-actions input.primary{background:var(--accent)}.config-grid{display:grid;grid-template-columns:1fr 1fr;gap:18px;align-items:start}.form-section{padding:25px;display:grid;gap:17px}.form-heading{font:700 2rem/1.1 Georgia,serif;border-bottom:1px solid var(--line);padding-bottom:14px}.form-title{font:700 3.5rem/1.1 Georgia,serif;margin:.25em 0}.muted{color:var(--muted)}.field{display:flex;flex-direction:column;gap:6px;font-size:.8rem;font-weight:750}.two{display:grid;grid-template-columns:1fr 1fr;gap:14px}.check{flex-direction:row;align-items:center}.check input{width:auto}.form-section small{font-weight:400;color:var(--muted)}.sticky-actions{grid-column:1/-1;display:flex;justify-content:flex-end;gap:10px}.primary{background:var(--accent)}.preview{margin-top:24px;padding:25px}.preview div{display:flex;justify-content:space-between;padding:10px;border-top:1px solid var(--line);gap:20px}.preview span{font-size:.8rem;color:var(--muted);white-space:nowrap}footer{text-align:center;border-top:1px solid var(--line);padding:30px;color:var(--muted);font-size:.8rem}.hero-title{font:700 clamp(2.5rem,6vw,5.5rem)/1.05 Georgia,serif;max-width:850px;margin:.12em 0}.hero-copy{font-size:1.2rem;color:var(--muted)}.section-title{font:700 2rem/1.1 Georgia,serif;margin-top:5px}.location-field{position:relative}.location-options{position:absolute;z-index:20;top:100%;left:0;right:0;background:white;border:1px solid var(--line);border-radius:8px;box-shadow:0 12px 30px #17211b26;overflow:hidden}.location-options button{display:block;width:100%;text-align:left;border-radius:0;background:white;color:var(--ink);font-size:.85rem;border-bottom:1px solid var(--line)}.location-options button:hover{background:#eee9de}.location-message{padding:12px;color:var(--muted);font-size:.8rem;font-weight:400}.tag-field{position:relative}.tag-toggle{width:100%;text-align:left;background:white;color:var(--ink);border:1px solid var(--line);font-weight:400}.tag-options{position:absolute;z-index:19;top:100%;left:0;right:0;min-width:190px;background:white;border:1px solid var(--line);border-radius:8px;box-shadow:0 12px 30px #17211b26;padding:8px}.tag-option{display:flex;flex-direction:row;align-items:center;gap:8px;padding:7px;border-radius:5px;cursor:pointer}.tag-option:hover{background:#eee9de}.tag-option input{width:auto}.range-field{position:relative}.view-tabs{display:flex;gap:8px;margin-bottom:16px}.tab{background:transparent;color:var(--green);border:1px solid var(--green)}.tab.active{background:var(--green);color:white}.view-panel{display:none}.view-panel.active{display:block}.search-filters{grid-template-columns:1.8fr 1.35fr 1fr .65fr auto;position:relative}.date-calendar{overflow:hidden;margin-bottom:18px}.calendar{overflow:hidden}.calendar-heading{font:700 1.6rem Georgia,serif;padding:22px}.calendar-tools{display:flex;align-items:center;justify-content:space-between;gap:12px;padding:0 22px 14px;color:var(--muted);font-size:.8rem}.clear-day{background:transparent;color:var(--green);border:1px solid var(--green);padding:7px 10px;font-size:.75rem}.weekdays,.calendar-grid{display:grid;grid-template-columns:repeat(7,1fr)}.weekdays{border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:9px 0;color:var(--muted);font-size:.75rem;font-weight:700;text-align:center}.weekdays>input{display:block;min-width:0;border:0;border-radius:0;padding:0;background:transparent;color:var(--muted);font-size:inherit;font-weight:inherit;text-align:center;opacity:1}.calendar-day{min-width:0;min-height:82px;padding:8px;border:0;border-right:1px solid var(--line);border-bottom:1px solid var(--line);border-radius:0;background:transparent;color:var(--ink);text-align:left;display:flex;flex-direction:column;justify-content:space-between}.calendar-day:nth-child(7n){border-right:0}.calendar-day.outside{opacity:.35}.calendar-day.has-events{background:#fff4e8}.calendar-day.has-events:hover,.calendar-day.selected{background:#f4d6c6}.calendar-day.range-start,.calendar-day.range-end{box-shadow:inset 0 0 0 2px var(--accent)}.calendar-day b{font-size:.7rem;color:var(--accent)}.day-events{display:grid;gap:10px;margin-top:18px}.tag-list{display:flex;gap:6px;flex-wrap:wrap;margin-top:10px}[hidden]{display:none!important}.event-row{padding:18px 22px;text-decoration:none;color:var(--ink);display:flex;justify-content:space-between;align-items:center}.event-row:hover{border-color:var(--accent)}.event-title{font:700 1.25rem Georgia,serif;margin:5px 0}.map{height:580px}.back-link{display:inline-block;margin-bottom:34px}.event-detail{display:grid;grid-template-columns:1.1fr .9fr;gap:45px;align-items:start}.detail-title{font:700 clamp(2.5rem,5vw,4.8rem)/1.05 Georgia,serif;margin:10px 0}.detail-time{font:700 2rem Georgia,serif;color:var(--accent)}.description{font-size:1.1rem}.detail-card{padding:22px;margin:30px 0}.detail-field{display:grid;grid-template-columns:90px minmax(0,1fr);align-items:start;gap:14px;padding:13px 0;border-bottom:1px solid var(--line)}.detail-field:last-child{border-bottom:0}.detail-label{font-size:.75rem;color:var(--muted);font-weight:700;text-transform:uppercase;letter-spacing:.08em}.detail-value{font-weight:700;overflow-wrap:anywhere}.hint{font-size:.75rem;color:var(--muted);font-weight:400}.detail-actions{display:flex;gap:10px}.detail-map{height:520px;position:sticky;top:24px}@media(max-width:800px){.search,.search-filters,.config-grid,.event-detail,.url-detect-row,.picker-selection{grid-template-columns:1fr}.picker-controls{align-items:stretch;flex-direction:column}.selector-picker{height:480px}.two{grid-template-columns:1fr}.sticky-actions{grid-column:auto}.section-head{align-items:flex-start;flex-direction:column}.source-title,.event-foot,.admin-event{align-items:flex-start;flex-direction:column}dl{flex-wrap:wrap}.search button{width:100%}nav{gap:10px;font-size:.85rem}.calendar-day{min-height:72px;padding:7px}.calendar-day b{font-size:.6rem}.weekdays{font-size:.6rem}.map,.detail-map{height:420px}.event-detail{gap:25px}} +:root{--ink:#17211b;--muted:#657068;--paper:#f4f0e7;--card:#fffdf7;--accent:#dd5b32;--green:#234c3c;--line:#d9d4c8}*{box-sizing:border-box}body{margin:0;background:var(--paper);color:var(--ink);font:16px/1.5 system-ui,-apple-system,sans-serif}a{color:var(--green)}header{height:68px;padding:0 max(24px,calc((100% - 1120px)/2));display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid var(--line)}nav{display:flex;gap:24px}nav a{text-decoration:none;font-weight:650}.brand{font:bold 24px Georgia,serif;text-decoration:none;color:var(--ink)}main{max-width:1120px;margin:auto;padding:48px 24px 90px}h1,h2,h3{font-family:Georgia,serif;line-height:1.1;margin:.25em 0}h1{font-size:clamp(2.5rem,6vw,5.5rem);max-width:850px}h2{font-size:2rem}.hero{padding:45px 0}.hero p:not(.eyebrow){font-size:1.2rem;color:var(--muted)}.eyebrow{font-size:.75rem;font-weight:800;letter-spacing:.16em;color:var(--accent)}.card{background:var(--card);border:1px solid var(--line);border-radius:12px;box-shadow:0 5px 18px #17211b0a}.search{display:grid;grid-template-columns:2fr 1.4fr 1.4fr .8fr auto;gap:12px;padding:18px;align-items:end}label{display:flex;flex-direction:column;gap:6px;font-size:.8rem;font-weight:750;position:relative}input,select{width:100%;border:1px solid var(--line);background:white;border-radius:7px;padding:12px;color:var(--ink);font:inherit}.suffix{position:absolute;right:12px;bottom:12px;color:var(--muted)}button,.button{border:0;border-radius:7px;padding:12px 18px;background:var(--green);color:white;text-decoration:none;font-weight:750;font:inherit;cursor:pointer;display:inline-block}.secondary{background:transparent;color:var(--green);border:1px solid var(--green)}.danger{background:#9d302c}.section-head{display:flex;justify-content:space-between;align-items:end;margin:52px 0 22px;gap:20px}.section-head h1{font-size:3.5rem}.section-head p{max-width:650px;color:var(--muted)}.events,.source-grid{display:grid;gap:14px}.event{padding:24px}.event h3{font-size:1.55rem}.event p{color:var(--muted)}.date{color:var(--accent);font-weight:800}.event-foot,.source-title,.actions{display:flex;align-items:center;justify-content:space-between;gap:12px;margin-top:22px}.pill{font-size:.75rem;background:#eee9de;padding:5px 9px;border-radius:99px}.good{background:#dce9df;color:#285039}.empty{text-align:center;padding:60px}.source{padding:24px}.source-stats{display:flex;gap:30px;flex-wrap:wrap;border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:15px 0;margin-top:18px}.source-stats>div{display:grid}.source-stats small{color:var(--muted)}.admin-events{display:grid;gap:12px}.admin-event{padding:22px;display:flex;align-items:center;justify-content:space-between;gap:20px}.admin-event.is-hidden{opacity:.6;border-style:dashed}.event-admin-form{padding:25px;display:grid;gap:18px}textarea{width:100%;border:1px solid var(--line);background:white;border-radius:7px;padding:12px;color:var(--ink);font:inherit;resize:vertical}.source-title{margin-top:0}.source-title a{word-break:break-all;font-size:.85rem}dl{display:flex;gap:40px;border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:15px 0}dt{color:var(--muted);font-size:.75rem}dd{margin:2px 0;font-weight:700}.actions{justify-content:flex-start}.actions form{margin:0}.notice{padding:12px 16px;border-radius:7px}.error{background:#f4deda;color:#792a27}.success{background:#dce9df;color:#285039}.url-detect-row{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:8px}.url-detect-row .detect-submit{width:auto;white-space:nowrap;background:var(--green);color:white;border:0;font-weight:700}.address-field{position:relative}.picker-panel{grid-column:1/-1;padding:25px;display:grid;gap:14px}.picker-controls{display:flex;align-items:end;gap:10px}.picker-controls .field{min-width:220px}.picker-steps{display:grid;grid-template-columns:repeat(4,1fr);gap:8px;margin:0;padding:0;list-style:none;counter-reset:picker-step}.picker-steps li{padding:10px;border:1px solid var(--line);border-radius:7px;color:var(--muted);font-size:.8rem}.picker-steps li:before{counter-increment:picker-step;content:counter(picker-step) ". ";font-weight:800}.picker-steps li.active{border-color:var(--accent);color:var(--ink);background:#fff4e8}.picker-steps li.complete{border-color:var(--green);color:var(--green);background:#dce9df}.picker-steps li span{font-weight:400}.picker-workspace{display:grid;grid-template-columns:minmax(0,2fr) minmax(260px,1fr);gap:14px;align-items:start}.selector-picker{width:100%;height:620px;border:1px solid var(--line);border-radius:8px;background:white}.picker-preview{height:620px;overflow:auto;border:1px solid var(--line);border-radius:8px;background:white;padding:14px}.picker-preview-head{display:flex;justify-content:space-between;gap:10px;padding-bottom:12px;border-bottom:1px solid var(--line)}.picker-preview-head span{color:var(--muted);font-size:.8rem}.picker-preview .notice{margin:12px 0;font-size:.8rem}.picker-preview-rows{display:grid}.picker-preview-row{display:grid;gap:3px;padding:11px 0;border-bottom:1px solid var(--line)}.picker-preview-row span{color:var(--muted);font-size:.8rem}.picker-selection{display:grid;grid-template-columns:minmax(0,1fr) minmax(0,1fr);align-items:start;gap:14px;padding:14px;background:#eee9de;border-radius:8px}.picker-selection>div{display:grid;gap:5px}.picker-selection code{overflow-wrap:anywhere}.picker-selection span{color:var(--muted);font-size:.8rem;overflow:hidden;text-overflow:ellipsis}.picker-actions{display:flex!important;flex-direction:row;flex-wrap:wrap}.picker-actions button{padding:9px 12px}.rules-details{grid-column:1/-1;overflow:hidden}.rules-details>summary{cursor:pointer;padding:22px;font:700 1.5rem Georgia,serif;display:flex;justify-content:space-between;gap:15px}.rules-details>summary span{font:400 .8rem system-ui;color:var(--muted)}.rules-details[open]>summary{border-bottom:1px solid var(--line)}.sticky-actions input[type=submit]{width:auto;background:var(--green);color:white;border:0;font-weight:700;cursor:pointer}.sticky-actions input.primary{background:var(--accent)}.config-grid{display:grid;grid-template-columns:1fr 1fr;gap:18px;align-items:start}.form-section{padding:25px;display:grid;gap:17px}.form-heading{font:700 2rem/1.1 Georgia,serif;border-bottom:1px solid var(--line);padding-bottom:14px}.form-title{font:700 3.5rem/1.1 Georgia,serif;margin:.25em 0}.muted{color:var(--muted)}.field{display:flex;flex-direction:column;gap:6px;font-size:.8rem;font-weight:750}.two{display:grid;grid-template-columns:1fr 1fr;gap:14px}.check{flex-direction:row;align-items:center}.check input{width:auto}.form-section small{font-weight:400;color:var(--muted)}.sticky-actions{grid-column:1/-1;display:flex;justify-content:flex-end;gap:10px}.primary{background:var(--accent)}.preview{margin-top:24px;padding:25px}.preview div{display:flex;justify-content:space-between;padding:10px;border-top:1px solid var(--line);gap:20px}.preview span{font-size:.8rem;color:var(--muted);white-space:nowrap}footer{text-align:center;border-top:1px solid var(--line);padding:30px;color:var(--muted);font-size:.8rem}.hero-title{font:700 clamp(2.5rem,6vw,5.5rem)/1.05 Georgia,serif;max-width:850px;margin:.12em 0}.hero-copy{font-size:1.2rem;color:var(--muted)}.section-title{font:700 2rem/1.1 Georgia,serif;margin-top:5px}.location-field{position:relative}.location-options{position:absolute;z-index:20;top:100%;left:0;right:0;background:white;border:1px solid var(--line);border-radius:8px;box-shadow:0 12px 30px #17211b26;overflow:hidden}.location-options button{display:block;width:100%;text-align:left;border-radius:0;background:white;color:var(--ink);font-size:.85rem;border-bottom:1px solid var(--line)}.location-options button:hover{background:#eee9de}.location-message{padding:12px;color:var(--muted);font-size:.8rem;font-weight:400}.tag-field{position:relative}.tag-toggle{width:100%;text-align:left;background:white;color:var(--ink);border:1px solid var(--line);font-weight:400}.tag-options{position:absolute;z-index:19;top:100%;left:0;right:0;min-width:190px;background:white;border:1px solid var(--line);border-radius:8px;box-shadow:0 12px 30px #17211b26;padding:8px}.tag-option{display:flex;flex-direction:row;align-items:center;gap:8px;padding:7px;border-radius:5px;cursor:pointer}.tag-option:hover{background:#eee9de}.tag-option input{width:auto}.range-field{position:relative}.view-tabs{display:flex;gap:8px;margin-bottom:16px}.tab{background:transparent;color:var(--green);border:1px solid var(--green)}.tab.active{background:var(--green);color:white}.view-panel{display:none}.view-panel.active{display:block}.search-filters{grid-template-columns:1.8fr 1.35fr 1fr .65fr auto;position:relative}.date-calendar{overflow:hidden;margin-bottom:18px}.calendar{overflow:hidden}.calendar-heading{font:700 1.6rem Georgia,serif;padding:22px}.calendar-tools{display:flex;align-items:center;justify-content:space-between;gap:12px;padding:0 22px 14px;color:var(--muted);font-size:.8rem}.clear-day{background:transparent;color:var(--green);border:1px solid var(--green);padding:7px 10px;font-size:.75rem}.weekdays,.calendar-grid{display:grid;grid-template-columns:repeat(7,1fr)}.weekdays{border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:9px 0;color:var(--muted);font-size:.75rem;font-weight:700;text-align:center}.weekdays>input{display:block;min-width:0;border:0;border-radius:0;padding:0;background:transparent;color:var(--muted);font-size:inherit;font-weight:inherit;text-align:center;opacity:1}.calendar-day{min-width:0;min-height:82px;padding:8px;border:0;border-right:1px solid var(--line);border-bottom:1px solid var(--line);border-radius:0;background:transparent;color:var(--ink);text-align:left;display:flex;flex-direction:column;justify-content:space-between}.calendar-day:nth-child(7n){border-right:0}.calendar-day.outside{opacity:.35}.calendar-day.has-events{background:#fff4e8}.calendar-day.has-events:hover,.calendar-day.selected{background:#f4d6c6}.calendar-day.range-start,.calendar-day.range-end{box-shadow:inset 0 0 0 2px var(--accent)}.calendar-day b{font-size:.7rem;color:var(--accent)}.day-events{display:grid;gap:10px;margin-top:18px}.tag-list{display:flex;gap:6px;flex-wrap:wrap;margin-top:10px}[hidden]{display:none!important}.event-row{padding:18px 22px;text-decoration:none;color:var(--ink);display:flex;justify-content:space-between;align-items:center}.event-row:hover{border-color:var(--accent)}.event-title{font:700 1.25rem Georgia,serif;margin:5px 0}.map{height:580px}.back-link{display:inline-block;margin-bottom:34px}.event-detail{display:grid;grid-template-columns:1.1fr .9fr;gap:45px;align-items:start}.detail-title{font:700 clamp(2.5rem,5vw,4.8rem)/1.05 Georgia,serif;margin:10px 0}.detail-time{font:700 2rem Georgia,serif;color:var(--accent)}.description{font-size:1.1rem}.detail-card{padding:22px;margin:30px 0}.detail-field{display:grid;grid-template-columns:90px minmax(0,1fr);align-items:start;gap:14px;padding:13px 0;border-bottom:1px solid var(--line)}.detail-field:last-child{border-bottom:0}.detail-label{font-size:.75rem;color:var(--muted);font-weight:700;text-transform:uppercase;letter-spacing:.08em}.detail-value{font-weight:700;overflow-wrap:anywhere}.hint{font-size:.75rem;color:var(--muted);font-weight:400}.detail-actions{display:flex;gap:10px}.detail-map{height:520px;position:sticky;top:24px}@media(max-width:800px){.search,.search-filters,.config-grid,.event-detail,.url-detect-row,.picker-selection,.picker-workspace{grid-template-columns:1fr}.picker-controls{align-items:stretch;flex-direction:column}.picker-steps{grid-template-columns:1fr 1fr}.selector-picker{height:480px}.picker-preview{height:auto;max-height:480px}.two{grid-template-columns:1fr}.sticky-actions{grid-column:auto}.section-head{align-items:flex-start;flex-direction:column}.source-title,.event-foot,.admin-event{align-items:flex-start;flex-direction:column}dl{flex-wrap:wrap}.search button{width:100%}nav{gap:10px;font-size:.85rem}.calendar-day{min-height:72px;padding:7px}.calendar-day b{font-size:.6rem}.weekdays{font-size:.6rem}.map,.detail-map{height:420px}.event-detail{gap:25px}} diff --git a/app/templates/source_form.html b/app/templates/source_form.html index 9935236..82975c3 100644 --- a/app/templates/source_form.html +++ b/app/templates/source_form.html @@ -12,7 +12,7 @@
Venue address
Latitude
Longitude
Timezone
-
Visual selector picker
Loads a sanitized, non-interactive copy of the downloaded HTML. Dynamic JavaScript content may not appear.
+
Visual selector picker
Loads a sanitized, non-interactive copy of the downloaded HTML. Dynamic JavaScript content may not appear.
over
Extraction strategy — advanced
HTML mode uses the convenient selector fields below. JSON and HTML-fragment modes use the generic JSON strategy directly.
Event container selector
Title selector
Date or range selector
Time selector
Link selector
Description selector
Location selector
Date format
diff --git a/compose.yaml b/compose.yaml index 6a221a8..1523060 100644 --- a/compose.yaml +++ b/compose.yaml @@ -12,5 +12,16 @@ services: - what2do-data:/data restart: unless-stopped + test: + build: . + working_dir: /app + command: ["sh", "-c", "pip install --no-cache-dir pytest==8.4.2 pytest-cov==6.2.1 && python -m pytest tests/unit -vv -rA --tb=short --cov=app --cov-report=term-missing --results-file=/test-results/latest.txt"] + volumes: + - ./test-results:/test-results + environment: + DATABASE_PATH: /tmp/what2do-test.db + PYTHONPATH: /app + profiles: ["test"] + volumes: what2do-data: diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..3a2bd06 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = tests +addopts = -ra -m "not network" +pythonpath = . +markers = + network: live-network integration tests, excluded from normal runs diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..42a6a26 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +pytest==8.4.2 +pytest-cov==6.2.1 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d192ce9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,112 @@ +import json +from pathlib import Path + +import pytest + + +def pytest_addoption(parser): + parser.addoption("--results-file", help="Write the human-readable terminal report to this file.") + + +def pytest_configure(config): + results_file = config.getoption("--results-file") + if results_file: + config.pluginmanager.register(_ResultsFilePlugin(Path(results_file)), "results-file") + + +class _ResultsFilePlugin: + def __init__(self, path): + self.path = path + self.reports = [] + + def pytest_runtest_logreport(self, report): + if report.when == "call" or report.failed: + self.reports.append(report) + + def pytest_sessionfinish(self, session, exitstatus): + lines = ["What2Do automated test results", "=" * 32, ""] + for report in self.reports: + outcome = report.outcome.upper() + lines.append(f"{outcome:7} {report.nodeid} ({report.duration:.3f}s)") + if report.failed: + lines.extend(("", str(report.longrepr), "")) + counts = {outcome: sum(report.outcome == outcome for report in self.reports) for outcome in ("passed", "failed", "skipped")} + lines.extend(("", "Summary", "-------", f"Passed: {counts['passed']}", f"Failed: {counts['failed']}", f"Skipped: {counts['skipped']}", f"Exit status: {exitstatus}")) + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text("\n".join(lines) + "\n") + + +@pytest.fixture +def source(): + return { + "url": "https://example.org/events", + "timezone": "Australia/Sydney", + "venue_name": "Example Venue", + "venue_address": "1 Example St, Wollongong NSW 2500", + "latitude": -34.42, + "longitude": 150.89, + } + + +@pytest.fixture +def fixture_html(): + def load(name): + return (Path(__file__).resolve().parent / "fixtures" / name).read_text() + return load + + +@pytest.fixture +def html_cards(fixture_html): + return fixture_html("event_cards.html") + + +class FakeResponse: + def __init__(self, *, text="", payload=None, status_code=200, url="https://example.org/events"): + self.text = text + self._payload = payload + self.status_code = status_code + self.url = url + self.content = text.encode() if payload is None else json.dumps(payload).encode() + + def json(self): + return self._payload + + def raise_for_status(self): + if self.status_code >= 400: + import httpx + raise httpx.HTTPStatusError("failed", request=httpx.Request("GET", self.url), response=httpx.Response(self.status_code)) + + +class RecordingClient: + def __init__(self, responses): + self.responses = list(responses) + self.calls = [] + + def request(self, method, url, **kwargs): + self.calls.append((method, url, kwargs)) + return self.responses.pop(0) + + +@pytest.fixture +def fake_response(): + return FakeResponse + + +@pytest.fixture +def recording_client(): + return RecordingClient + + +@pytest.fixture +def temp_db(tmp_path, monkeypatch): + import app.db as db_module + path = tmp_path / "nested" / "what2do.db" + monkeypatch.setattr(db_module, "DB_PATH", str(path)) + db_module.init_db() + return path + + +@pytest.fixture +def source_manifest(): + path = Path(__file__).parent / "fixtures" / "illawarra_sources.json" + return json.loads(path.read_text()) diff --git a/tests/fixtures/event_cards.html b/tests/fixtures/event_cards.html new file mode 100644 index 0000000..6abb796 --- /dev/null +++ b/tests/fixtures/event_cards.html @@ -0,0 +1,19 @@ + + + +
="7 September 2026"> + Event One + + 7:30pm +

First description

+ Town Hall +
+
+ Event Two + + 8:00pm +

Second description

+ Art Gallery +
+ + diff --git a/tests/fixtures/hostile_snapshot.html b/tests/fixtures/hostile_snapshot.html new file mode 100644 index 0000000..2023178 --- /dev/null +++ b/tests/fixtures/hostile_snapshot.html @@ -0,0 +1,15 @@ + + + + + + + + +
+ + Event + +

Safe

+ + diff --git a/tests/fixtures/illawarra_sources.json b/tests/fixtures/illawarra_sources.json new file mode 100644 index 0000000..416410b --- /dev/null +++ b/tests/fixtures/illawarra_sources.json @@ -0,0 +1,52 @@ +[ + {"name":"Merrigong Theatre Company","url":"https://merrigong.com.au/whats-on/","mode":"html"}, + {"name":"Anita's Theatre","url":"https://www.anitastheatre.com.au/whats-on/","mode":"dynamic"}, + {"name":"La La La's","url":"https://www.lalalas.com.au/all-events/","mode":"html"}, + {"name":"UOW Events","url":"https://www.uow.edu.au/events/","mode":"html"}, + {"name":"Wollongong Art Gallery","url":"https://wollongongartgallery.au/whats-on/events","mode":"html"}, + {"name":"HOYTS Warrawong","url":"https://www.hoyts.com.au/cinemas/warrawong","mode":"dynamic"}, + {"name":"Wests Illawarra","url":"https://www.westsillawarra.com.au/whats-on/","mode":"html"}, + {"name":"Towradgi Beach Hotel","url":"https://towradgibeachhotel.com.au/whats-on/","mode":"html"}, + {"name":"Club Windang calendar","url":"https://clubwindang.com.au/events/","mode":"html"}, + {"name":"Club Windang recurring events","url":"https://clubwindang.com.au/upcoming-events/","mode":"recurring"}, + {"name":"Resin Brewing","url":"https://resinbrewing.com.au/pages/events","mode":"html"}, + {"name":"Collegians Wollongong Entertainment","url":"https://collegians.com.au/entertainment/","mode":"html"}, + {"name":"Collegians Balgownie Entertainment","url":"https://collegians.com.au/balgownie-entertainment/","mode":"html"}, + {"name":"Wollongong City Events","url":"https://www.wollongong.nsw.gov.au/community/events-and-programs/events","mode":"html"}, + {"name":"Wollongong City Libraries","url":"https://www.wollongong.nsw.gov.au/library/whats-on","mode":"html"}, + {"name":"Wollongong Library event search","url":"https://www.wollongong.nsw.gov.au/library/whats-on/events","mode":"html"}, + {"name":"Destination Wollongong","url":"https://www.visitwollongong.com.au/events/","mode":"dynamic"}, + {"name":"UniBar UOW","url":"https://unibar.uow.edu.au/gigs-events/","mode":"html"}, + {"name":"Pinot & Picasso Wollongong","url":"https://pinotandpicasso.com.au/wollongong/","mode":"dynamic"}, + {"name":"Heyday","url":"https://heyday.bar/events/","mode":"mixed"}, + {"name":"Club Thirroul","url":"https://www.clubthirroul.com.au/whats-on/","mode":"html"}, + {"name":"Thirroul District","url":"https://thirrouldistrict.com.au/whats-on/","mode":"html"}, + {"name":"Thirroul Library","url":"https://www.wollongong.nsw.gov.au/library/your-library/locations-and-hours/thirroul-library","mode":"html"}, + {"name":"Headlands Austinmer","url":"https://headlandshotel.com.au/whats-on/","mode":"html"}, + {"name":"South Coast Writers Centre","url":"https://southcoastwriters.org/upcomingevents","mode":"html"}, + {"name":"Coledale Community Hall","url":"https://southcoastwriters.org/coledale-community-hall-events","mode":"recurring"}, + {"name":"Heritage Hotel Bulli","url":"https://heritagehotel.com.au/whats-on/","mode":"html"}, + {"name":"Bulli District","url":"https://bullidistrict.org.au/events","mode":"html"}, + {"name":"Corrimal Library","url":"https://www.wollongong.nsw.gov.au/library/your-library/locations-and-hours/corrimal-library","mode":"html"}, + {"name":"Corrimal RSL","url":"https://www.corrimalrsl.com.au/whats-on/","mode":"html"}, + {"name":"Helensburgh Library","url":"https://www.wollongong.nsw.gov.au/library/your-library/locations-and-hours/helensburgh-library","mode":"html"}, + {"name":"Helensburgh Hotel","url":"https://www.helensburghhotel.com.au/whats-on","mode":"recurring"}, + {"name":"Illawarra Folk Club","url":"https://www.illawarrafolkclub.org.au/","mode":"dynamic"}, + {"name":"Illawarra Folk Festival related events","url":"https://www.illawarrafolkfestival.com.au/moreevents","mode":"html"}, + {"name":"Bulli Workers Club","url":"https://www.bulliworkers.com.au/whats-on/","mode":"html"}, + {"name":"Shellharbour Council","url":"https://www.shellharbour.nsw.gov.au/whats-on","mode":"html"}, + {"name":"Visit Shellharbour","url":"https://www.visitshellharbour.com.au/whats-on/","mode":"dynamic"}, + {"name":"Shellharbour Civic Centre","url":"https://www.shellharbourciviccentre.com.au/whats-on/all-events/","mode":"dynamic"}, + {"name":"Shellharbour Libraries","url":"https://libraries.shellharbour.nsw.gov.au/whats-on","mode":"html"}, + {"name":"Shellharbour Museum","url":"https://museum.shellharbour.nsw.gov.au/programs/","mode":"recurring"}, + {"name":"Warilla Bowls events","url":"https://warillabowls.com.au/events","mode":"html"}, + {"name":"Warilla Bowls club activities","url":"https://warillabowls.com.au/club-activities","mode":"recurring"}, + {"name":"Dapto Leagues Club","url":"https://daptoleagues.com.au/live-music/","mode":"recurring"}, + {"name":"Waterfront Tavern","url":"https://waterfronttavern.com.au/whats-on/","mode":"html"}, + {"name":"Kiama Bowling Club","url":"https://www.kiamabowling.com.au/whats-on/","mode":"image-heavy"}, + {"name":"Pavilion Kiama","url":"https://www.thepavilionkiama.com.au/whats-on/","mode":"html"}, + {"name":"The Vault Port Kembla","url":"https://www.thevaultportkembla.com/events","mode":"dynamic"}, + {"name":"Kiama Council Events","url":"https://www.kiama.nsw.gov.au/Events","mode":"html"}, + {"name":"Kiama Tourism Events","url":"https://kiama.com.au/events/","mode":"dynamic"}, + {"name":"Museums & Galleries NSW","url":"https://mgnsw.org.au/events/","mode":"html"} +] diff --git a/tests/integration/test_live_sources.py b/tests/integration/test_live_sources.py new file mode 100644 index 0000000..094ab96 --- /dev/null +++ b/tests/integration/test_live_sources.py @@ -0,0 +1,17 @@ +import pytest + +from app.crawler import _http_client + + +pytestmark = pytest.mark.network + + +@pytest.mark.parametrize("source_index", range(50)) +def test_source_returns_a_response(source_manifest, source_index): + source = source_manifest[source_index] + with _http_client(timeout=30) as client: + response = client.get(source["url"]) + assert response.status_code < 500, f"{source['name']} returned {response.status_code}" + assert response.content, f"{source['name']} returned an empty response" + content_type = response.headers.get("content-type", "") + assert any(value in content_type.lower() for value in ("html", "json", "xml", "text")), content_type diff --git a/tests/unit/test_calendar_and_sanitizer.py b/tests/unit/test_calendar_and_sanitizer.py new file mode 100644 index 0000000..594a148 --- /dev/null +++ b/tests/unit/test_calendar_and_sanitizer.py @@ -0,0 +1,77 @@ +from datetime import datetime, timezone + +from bs4 import BeautifulSoup + +from app.main import ( + PICKER_SCRIPT, + calendar_day_counts, + event_overlaps_range, + sanitize_picker_html, +) + + +def test_multiday_events_count_each_overlapping_calendar_day(): + events = [ + {"day_key": "2026-09-02", "end_day_key": "2026-09-04"}, + {"day_key": "2026-09-04", "end_day_key": "2026-09-04"}, + ] + assert calendar_day_counts(events) == { + "2026-09-02": 1, + "2026-09-03": 1, + "2026-09-04": 2, + } + + +def test_date_range_overlap_semantics_are_end_exclusive_for_query_range(): + event_start = datetime(2026, 9, 2, tzinfo=timezone.utc) + event_end = datetime(2026, 9, 13, tzinfo=timezone.utc) + assert event_overlaps_range( + event_start, + event_end, + datetime(2026, 9, 5, tzinfo=timezone.utc), + datetime(2026, 9, 7, tzinfo=timezone.utc), + ) + assert not event_overlaps_range( + event_start, + event_end, + datetime(2026, 9, 14, tzinfo=timezone.utc), + datetime(2026, 9, 20, tzinfo=timezone.utc), + ) + assert not event_overlaps_range( + datetime(2026, 9, 20, tzinfo=timezone.utc), + None, + datetime(2026, 9, 14, tzinfo=timezone.utc), + datetime(2026, 9, 20, tzinfo=timezone.utc), + ) + + +def test_sanitizer_removes_active_content_and_rewrites_assets(fixture_html): + output = sanitize_picker_html(fixture_html("hostile_snapshot.html"), "https://example.org/events/") + soup = BeautifulSoup(output, "html.parser") + assert soup.select_one("form, iframe, base") is None + assert "unsafe" not in output + assert "onload=" not in output and "onclick=" not in output + assert "href" not in soup.select_one("a").attrs + assert soup.select_one("img")["src"] == "https://example.org/image.jpg" + assert soup.select_one('link[rel="stylesheet"]')["href"] == "https://example.org/site.css" + assert "what2do-selector" in output + + +def test_picker_script_contract_contains_generalization_fields(): + for field in ("containerSelector", "containerCount", "relativeSelector", "generalizedSelector", "selectorCount"): + assert field in PICKER_SCRIPT + assert "idPattern" in PICKER_SCRIPT + assert "[id^=" in PICKER_SCRIPT + + +def test_picker_script_supports_guided_highlights_and_preview(): + for contract in ("what2do-picker-config", "what2do-picker-preview", "inconsistencies", "withinConfiguredContainer"): + assert contract in PICKER_SCRIPT + for highlight in ("w2d-container", "w2d-field"): + assert highlight in PICKER_SCRIPT + + +def test_sanitizer_preserves_selector_relevant_ids_and_classes(): + output = sanitize_picker_html('

One

', "https://example.org") + soup = BeautifulSoup(output, "html.parser") + assert soup.select_one("#post-12345.event.card .title").get_text() == "One" diff --git a/tests/unit/test_crawler_dates.py b/tests/unit/test_crawler_dates.py new file mode 100644 index 0000000..1193a2b --- /dev/null +++ b/tests/unit/test_crawler_dates.py @@ -0,0 +1,61 @@ +from datetime import datetime +from zoneinfo import ZoneInfo + +import pytest + +from app.crawler import _parse_date, _parse_date_range, _weekly_dates, external_key, ExtractedEvent + + +NOW = datetime(2026, 9, 7, 12, 0, tzinfo=ZoneInfo("Australia/Sydney")) +SOURCE = {"timezone": "Australia/Sydney", "date_format": ""} + + +def test_parses_naive_australian_date_with_timezone(): + value = _parse_date("7 September 2026 7:30pm", SOURCE, NOW) + assert value.isoformat() == "2026-09-07T19:30:00+10:00" + + +def test_parses_explicit_format_without_year_to_next_occurrence(): + value = _parse_date("4 Sep 19:30", {**SOURCE, "date_format": "%d %b %H:%M"}, NOW) + assert value.year == 2027 + + +@pytest.mark.parametrize("separator", ["-", "–", "—"]) +def test_same_month_range(separator): + start, end = _parse_date_range(f"2 {separator} 13 Sep 2026", SOURCE, NOW) + assert (start.year, start.month, start.day) == (2026, 9, 2) + assert (end.year, end.month, end.day) == (2026, 9, 13) + + +def test_cross_month_range(): + start, end = _parse_date_range("30 Sep - 4 Oct 2026", SOURCE, NOW) + assert (start.month, start.day) == (9, 30) + assert (end.month, end.day) == (10, 4) + + +def test_non_range_has_no_end(): + start, end = _parse_date_range("9 September 2026 8pm", SOURCE, NOW) + assert start.hour == 20 + assert end is None + + +def test_rejects_relative_only_and_implausible_dates(): + with pytest.raises(ValueError): + _parse_date("next Friday", SOURCE, NOW) + with pytest.raises(ValueError): + _parse_date("13 Sep 2002", SOURCE, NOW) + + +def test_weekly_dates_generate_twelve_occurrences(): + values = _weekly_dates("Every Friday", NOW) + assert len(values) == 12 + assert values[0].weekday() == 4 + assert all((right - left).days == 7 for left, right in zip(values, values[1:])) + + +def test_external_key_is_case_insensitive_but_time_sensitive(): + first = ExtractedEvent("Trivia", NOW, address="Town Hall") + same = ExtractedEvent("TRIVIA", NOW, address="TOWN HALL") + later = ExtractedEvent("Trivia", NOW.replace(hour=13), address="Town Hall") + assert external_key(first) == external_key(same) + assert external_key(first) != external_key(later) diff --git a/tests/unit/test_crawler_extraction.py b/tests/unit/test_crawler_extraction.py new file mode 100644 index 0000000..69696de --- /dev/null +++ b/tests/unit/test_crawler_extraction.py @@ -0,0 +1,63 @@ +import json + +import pytest + +from app.crawler import detect_rules, extract_events + + +def base_source(**overrides): + value = {"url": "https://example.org/events", "timezone": "Australia/Sydney", "item_selector": "json-ld", "title_selector": "", "date_selector": "", "time_selector": "", "link_selector": "", "description_selector": "", "location_selector": "", "date_format": "", "recurring_weekly": 0} + value.update(overrides) + return value + + +def test_jsonld_extracts_graph_event_with_end_address_and_relative_url(): + payload = {"@graph": [{"@type": "Event", "name": "Gallery Talk", "startDate": "2026-09-07T18:00:00+10:00", "endDate": "2026-09-07T20:00:00+10:00", "description": "A talk", "url": "/talk", "location": {"address": {"streetAddress": "1 Art St", "addressLocality": "Wollongong", "addressRegion": "NSW", "postalCode": "2500"}}}]} + html = f'' + event = extract_events(html, base_source())[0] + assert event.title == "Gallery Talk" + assert event.ends_at.hour == 20 + assert event.address == "1 Art St, Wollongong, NSW, 2500" + assert event.url == "https://example.org/talk" + + +def test_jsonld_ignores_invalid_and_missing_dates(): + html = ''' + ''' + events = extract_events(html, base_source()) + assert [event.title for event in events] == ["Good"] + + +def test_html_extraction_skips_invalid_sibling(): + html = '''

Bad

+ ''' + source = base_source(item_selector="article", title_selector="h2", date_selector="time", link_selector="a") + events = extract_events(html, source) + assert len(events) == 1 + assert events[0].title == "Good" + + +def test_detector_prefers_jsonld(): + html = '' + rules, events, message = detect_rules(html, "https://example.org/events") + assert rules["item_selector"] == "json-ld" + assert len(events) == 1 + assert "JSON-LD" in message + + +def test_detector_finds_repeated_html_cards(): + html = '''

One

More
+

Two

More
''' + rules, events, message = detect_rules(html, "https://example.org/events") + assert len(events) >= 2 + assert rules["item_selector"] + assert rules["title_selector"] + assert rules["date_selector"] + assert "Guessed CSS" in message + + +def test_detector_rejects_social_feed_false_positive(): + html = '''
Party this FridayFriday night
+
More next FridayFriday night
''' + with pytest.raises(Exception, match="No repeated event-like"): + detect_rules(html, "https://example.org/events") diff --git a/tests/unit/test_db_and_crawl.py b/tests/unit/test_db_and_crawl.py new file mode 100644 index 0000000..51cee32 --- /dev/null +++ b/tests/unit/test_db_and_crawl.py @@ -0,0 +1,127 @@ +import json +import sqlite3 +from contextlib import contextmanager +from datetime import datetime +from zoneinfo import ZoneInfo + +from app.crawler import ExtractedEvent, external_key + + +def insert_source(db, strategy_json='{}'): + cursor = db.execute("""INSERT INTO sources(name,url,venue_name,venue_address,timezone,strategy_json,item_selector,title_selector,date_selector) + VALUES('Test','https://example.org/events','Venue','1 Test St','Australia/Sydney',?,'.event','h2','time')""", (strategy_json,)) + return cursor.lastrowid + + +def test_fresh_database_and_idempotent_initialization(temp_db): + import app.db as module + module.init_db() + with module.connect() as db: + tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} + assert {"sources", "events", "geocode_cache"} <= tables + assert "strategy_json" in {row["name"] for row in db.execute("PRAGMA table_info(sources)")} + assert "hidden" in {row["name"] for row in db.execute("PRAGMA table_info(events)")} + + +def test_source_delete_cascades_events(temp_db): + import app.db as module + with module.connect() as db: + source_id = insert_source(db) + db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url) VALUES(?,?,?,?,?,?,?)", (source_id, "key", "One", "2026-09-07T10:00:00+00:00", "Venue", "Address", "https://example.org/one")) + db.execute("DELETE FROM sources WHERE id=?", (source_id,)) + assert db.execute("SELECT count(*) FROM events").fetchone()[0] == 0 + + +def test_legacy_empty_strategy_is_migrated(temp_db): + import app.db as module + with module.connect() as db: + source_id = insert_source(db, '{}') + module.init_db() + with module.connect() as db: + strategy = json.loads(db.execute("SELECT strategy_json FROM sources WHERE id=?", (source_id,)).fetchone()[0]) + assert strategy["mode"] == "html" + assert strategy["item_selector"] == ".event" + + +def test_crawl_deduplicates_and_preserves_hidden(temp_db, monkeypatch): + import app.db as db_module + import app.main as main + monkeypatch.setattr(main, "connect", db_module.connect) + with db_module.connect() as db: + source_id = insert_source(db, '{"mode":"html"}') + start = datetime(2026, 9, 7, 19, 0, tzinfo=ZoneInfo("Australia/Sydney")) + extracted = ExtractedEvent("Trivia", start, description="First", url="https://example.org/trivia", address="1 Test St") + + class ClientContext: + def __enter__(self): return object() + def __exit__(self, *args): return None + + monkeypatch.setattr(main, "_http_client", lambda timeout=45: ClientContext()) + monkeypatch.setattr(main, "execute_strategy", lambda source, client: [extracted, extracted]) + assert main.crawl_source(source_id) == 2 + with db_module.connect() as db: + rows = db.execute("SELECT * FROM events").fetchall() + assert len(rows) == 1 + event_id = rows[0]["id"] + db.execute("UPDATE events SET hidden=1 WHERE id=?", (event_id,)) + extracted.description = "Updated" + main.crawl_source(source_id) + with db_module.connect() as db: + row = db.execute("SELECT * FROM events WHERE id=?", (event_id,)).fetchone() + assert row["description"] == "Updated" + assert row["hidden"] == 1 + + +def test_successful_empty_crawl_removes_all_stale_events(temp_db, monkeypatch): + import app.db as db_module + import app.main as main + monkeypatch.setattr(main, "connect", db_module.connect) + with db_module.connect() as db: + source_id = insert_source(db, '{"mode":"html"}') + db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url,hidden) VALUES(?,?,?,?,?,?,?,?)", (source_id, "stale", "Old event", "2026-09-07T10:00:00+00:00", "Venue", "Address", "https://example.org/old", 1)) + + class ClientContext: + def __enter__(self): return object() + def __exit__(self, *args): return None + + monkeypatch.setattr(main, "_http_client", lambda timeout=45: ClientContext()) + monkeypatch.setattr(main, "execute_strategy", lambda source, client: []) + assert main.crawl_source(source_id) == 0 + with db_module.connect() as db: + assert db.execute("SELECT count(*) FROM events WHERE source_id=?", (source_id,)).fetchone()[0] == 0 + + +def test_failed_crawl_preserves_existing_events(temp_db, monkeypatch): + import app.db as db_module + import app.main as main + monkeypatch.setattr(main, "connect", db_module.connect) + with db_module.connect() as db: + source_id = insert_source(db, '{"mode":"html"}') + db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url) VALUES(?,?,?,?,?,?,?)", (source_id, "existing", "Existing event", "2026-09-07T10:00:00+00:00", "Venue", "Address", "https://example.org/existing")) + + class ClientContext: + def __enter__(self): return object() + def __exit__(self, *args): return None + + monkeypatch.setattr(main, "_http_client", lambda timeout=45: ClientContext()) + monkeypatch.setattr(main, "execute_strategy", lambda source, client: (_ for _ in ()).throw(RuntimeError("crawl failed"))) + try: + main.crawl_source(source_id) + assert False, "Expected crawl failure" + except RuntimeError: + pass + with db_module.connect() as db: + assert db.execute("SELECT count(*) FROM events WHERE source_id=?", (source_id,)).fetchone()[0] == 1 + + +def test_external_key_database_constraint(temp_db): + import app.db as module + with module.connect() as db: + source_id = insert_source(db) + values = (source_id, "same", "One", "2026-09-07T10:00:00+00:00", "Venue", "Address", "https://example.org/one") + db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url) VALUES(?,?,?,?,?,?,?)", values) + try: + db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url) VALUES(?,?,?,?,?,?,?)", values) + assert False, "Expected unique constraint" + except sqlite3.IntegrityError: + pass diff --git a/tests/unit/test_manifest.py b/tests/unit/test_manifest.py new file mode 100644 index 0000000..2cd7125 --- /dev/null +++ b/tests/unit/test_manifest.py @@ -0,0 +1,21 @@ +from urllib.parse import urlparse + + +def test_illawarra_manifest_contains_fifty_unique_https_sources(source_manifest): + assert len(source_manifest) == 50 + urls = [item["url"] for item in source_manifest] + assert len(set(urls)) == 50 + assert all(urlparse(url).scheme == "https" for url in urls) + + +def test_updated_collegians_urls_are_present(source_manifest): + urls = {item["url"] for item in source_manifest} + assert "https://collegians.com.au/entertainment/" in urls + assert "https://collegians.com.au/balgownie-entertainment/" in urls + assert "https://www.collegians.com.au/whats-on/wollongong/" not in urls + assert "https://www.collegians.com.au/whats-on/balgownie/" not in urls + + +def test_manifest_has_expected_mode_diversity(source_manifest): + modes = {item["mode"] for item in source_manifest} + assert {"html", "dynamic", "recurring", "mixed", "image-heavy"} <= modes diff --git a/tests/unit/test_strategies.py b/tests/unit/test_strategies.py new file mode 100644 index 0000000..854c7dc --- /dev/null +++ b/tests/unit/test_strategies.py @@ -0,0 +1,106 @@ +import json +from datetime import datetime + +import pytest + +from app.strategies import ( + _events_from_html, + _events_from_json, + execute_strategy, + json_path, + parse_strategy, +) + + +def test_parse_strategy_deep_copies_dict(): + original = {"mode": "html", "body": {"page": 1}} + parsed = parse_strategy(original) + parsed["body"]["page"] = 2 + assert original["body"]["page"] == 1 + assert parse_strategy(json.dumps(original)) == original + assert parse_strategy("not json") == {} + + +def test_json_path_handles_dicts_lists_and_defaults(): + value = {"results": [{"title": "One"}]} + assert json_path(value, "results.0.title") == "One" + assert json_path(value, "results.2.title", "missing") == "missing" + assert json_path(value, "") is value + + +def test_html_strategy_extracts_text_attributes_and_links(source, html_cards): + strategy = { + "item_selector": ".event", + "title_selector": ".title", + "date_selector": "time", + "date_attribute": "datetime", + "link_selector": ".title", + "link_attribute": "href", + "description_selector": ".description", + "location_selector": ".venue", + } + events = _events_from_html(html_cards, strategy, source) + assert [event.title for event in events] == ["Event One", "Event Two"] + assert events[0].starts_at.hour == 19 + assert events[0].url == "https://example.org/events/one" + assert events[0].description == "First description" + assert events[0].address == "Town Hall" + + +def test_html_scope_attribute_and_duration(source): + html = '

One

' + strategy = {"item_selector": ".event", "title_selector": "h2", "date_selector": ":scope", "date_attribute": "data-date", "duration_minutes": 90} + event = _events_from_html(html, strategy, source)[0] + assert (event.ends_at - event.starts_at).total_seconds() == 5400 + + +def test_html_range_populates_end(source): + html = '

Festival

' + event = _events_from_html(html, {"item_selector": ".event", "title_selector": "h2", "date_selector": "time"}, source)[0] + assert event.starts_at.day == 2 + assert event.ends_at.day == 13 + + +def test_json_strategy_extracts_nested_fields(source): + payload = {"results": [{"title": "Talk", "start": "2026-09-10T18:00:00+10:00", "end": "2026-09-10T20:00:00+10:00", "venue": {"address": "Library"}, "url": "/talk"}]} + strategy = {"items_path": "results", "title_path": "title", "date_path": "start", "end_path": "end", "location_path": "venue.address", "link_path": "url"} + event = _events_from_json(payload, strategy, source)[0] + assert event.title == "Talk" + assert event.ends_at.hour == 20 + assert event.address == "Library" + + +def test_json_join_supplies_metadata_and_duration(source): + payload = [{"movieId": "M1", "utcDate": "2026-09-11T10:00:00+00:00", "link": "/book"}] + strategy = {"items_path": "", "date_path": "utcDate", "link_path": "link", "join_item_key": "movieId", "join_title_path": "name", "join_description_path": "summary", "join_duration_path": "duration"} + event = _events_from_json(payload, strategy, source, {"M1": {"name": "Movie", "summary": "Plot", "duration": 120}})[0] + assert event.title == "Movie" + assert event.description == "Plot" + assert (event.ends_at - event.starts_at).total_seconds() == 7200 + + +def test_disabled_json_item_is_skipped(source): + payload = [{"title": "Hidden", "start": "2026-09-10", "disabled": True}] + strategy = {"items_path": "", "title_path": "title", "date_path": "start", "disabled_path": "disabled"} + assert _events_from_json(payload, strategy, source) == [] + + +def test_html_fragment_paginates(source, fake_response, recording_client, monkeypatch): + import app.strategies as module + monkeypatch.setattr(module, "validate_public_url", lambda url: None) + responses = [ + fake_response(payload={"html": '

One

', "pages": 2}), + fake_response(payload={"html": '

Two

', "pages": 2}), + ] + client = recording_client(responses) + source = {**source, "strategy_json": json.dumps({"mode": "html_fragment", "request_url": "https://example.org/ajax", "method": "POST", "body": {"action": "events"}, "html_path": "html", "item_selector": ".event", "title_selector": "h2", "date_selector": "time", "page_parameter": "page", "max_pages": 5, "total_pages_path": "pages"})} + events = execute_strategy(source, client) + assert [event.title for event in events] == ["One", "Two"] + assert client.calls[0][2]["data"]["page"] == "1" + assert client.calls[1][2]["data"]["page"] == "2" + + +def test_unsupported_mode_raises(source, recording_client): + source = {**source, "strategy_json": '{"mode":"magic"}'} + with pytest.raises(Exception, match="Unsupported extraction mode"): + execute_strategy(source, recording_client([]))