Add automated regression suite and test reports
This commit is contained in:
parent
8267064bc1
commit
7ed76e54da
9
.gitignore
vendored
Normal file
9
.gitignore
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
test-results/*.txt
|
||||
.venv/
|
||||
data/
|
||||
*.db
|
||||
@ -4,7 +4,9 @@ WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY app ./app
|
||||
COPY README.md .
|
||||
COPY tests ./tests
|
||||
COPY pytest.ini .
|
||||
COPY README.md STRATEGIES.md ./
|
||||
RUN mkdir -p /data
|
||||
EXPOSE 8000
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
||||
16
README.md
16
README.md
@ -35,6 +35,22 @@ Use **Test extraction** to preview up to 20 records before saving. Static server
|
||||
|
||||
Addresses are geocoded and cached using OpenStreetMap's public Nominatim service. The search field also accepts `latitude,longitude` directly, and every source has optional coordinate fields for deterministic/offline venue positioning. Set a real contact in `NOMINATIM_USER_AGENT` before public/shared use and follow the [Nominatim usage policy](https://operations.osmfoundation.org/policies/nominatim/). For heavier use, configure or self-host a compatible geocoder.
|
||||
|
||||
## Tests
|
||||
|
||||
The approved 50-source Illawarra corpus is stored in `tests/fixtures/illawarra_sources.json`. Normal tests are fixture-based and do not access live websites:
|
||||
|
||||
```sh
|
||||
docker compose --profile test run --rm test
|
||||
```
|
||||
|
||||
Detailed per-test outcomes, durations, failure traces, and totals are exported to `test-results/latest.txt` after every run. The file is replaced on the next run and is ignored by Git.
|
||||
|
||||
Run optional live reachability checks separately (results depend on third-party availability and access policies):
|
||||
|
||||
```sh
|
||||
docker compose --profile test run --rm test pytest -m network tests/integration
|
||||
```
|
||||
|
||||
## Operational notes
|
||||
|
||||
- SQLite data is persisted in the `what2do-data` Docker volume.
|
||||
|
||||
@ -105,15 +105,17 @@ def _parse_date(value: str, source: dict, now: datetime) -> datetime:
|
||||
candidates = [fmt]
|
||||
# Event lists commonly omit a year. Resolve to the next occurrence.
|
||||
for candidate in candidates:
|
||||
parsed = datetime.strptime(value, candidate).replace(tzinfo=tz)
|
||||
if "%Y" not in candidate and "%y" not in candidate:
|
||||
parsed = parsed.replace(year=now.year)
|
||||
has_year = "%Y" in candidate or "%y" in candidate
|
||||
parse_value = value if has_year else f"{value} {now.year}"
|
||||
parse_format = candidate if has_year else f"{candidate} %Y"
|
||||
parsed = datetime.strptime(parse_value, parse_format).replace(tzinfo=tz)
|
||||
if not has_year:
|
||||
if parsed < now - timedelta(days=2):
|
||||
parsed = parsed.replace(year=now.year + 1)
|
||||
return parsed
|
||||
if not re.search(r"\b\d{1,4}\b", value):
|
||||
raise ValueError("Date text must contain a numeric day or year")
|
||||
parsed = date_parser.parse(value, fuzzy=True, default=now.replace(tzinfo=None), tzinfos={"INFO": None, "IPAC": None, "IMAX": None})
|
||||
parsed = date_parser.parse(value, fuzzy=True, default=now.replace(tzinfo=None, second=0, microsecond=0), tzinfos={"INFO": None, "IPAC": None, "IMAX": None})
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=tz)
|
||||
if not now.year - 1 <= parsed.year <= now.year + 10:
|
||||
|
||||
75
app/main.py
75
app/main.py
@ -149,6 +149,8 @@ def crawl_source(source_id: int):
|
||||
if seen:
|
||||
placeholders = ",".join("?" for _ in seen)
|
||||
db.execute(f"DELETE FROM events WHERE source_id=? AND external_key NOT IN ({placeholders})", (source_id, *seen))
|
||||
else:
|
||||
db.execute("DELETE FROM events WHERE source_id=?", (source_id,))
|
||||
db.execute("UPDATE sources SET last_crawled_at=CURRENT_TIMESTAMP,last_status=?,last_error=NULL WHERE id=?", (f"Found {len(events)} events", source_id))
|
||||
return len(events)
|
||||
except Exception as exc:
|
||||
@ -189,6 +191,22 @@ def distance_km(lat1, lon1, lat2, lon2):
|
||||
a = math.sin(dp/2)**2 + math.cos(p1)*math.cos(p2)*math.sin(dl/2)**2
|
||||
return 6371 * 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
|
||||
|
||||
|
||||
def event_overlaps_range(event_start: datetime, event_end: datetime | None, range_start: datetime, range_end: datetime) -> bool:
|
||||
return event_start < range_end and (event_end or event_start) >= range_start
|
||||
|
||||
|
||||
def calendar_day_counts(events: list[dict]) -> dict[str, int]:
|
||||
counts = {}
|
||||
for event in events:
|
||||
covered_day = datetime.fromisoformat(event["day_key"]).date()
|
||||
final_day = datetime.fromisoformat(event["end_day_key"]).date()
|
||||
while covered_day <= final_day:
|
||||
key = covered_day.isoformat()
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
covered_day += timedelta(days=1)
|
||||
return counts
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index(request: Request, location: str = "", latitude: float | None = None, longitude: float | None = None, start: str = "", end: str = "", radius: float = 25, tag: list[str] = Query(default=[]), term: str = ""):
|
||||
point = (latitude, longitude, location) if latitude is not None and longitude is not None else (geocode(location) if location else None)
|
||||
@ -230,14 +248,7 @@ def index(request: Request, location: str = "", latitude: float | None = None, l
|
||||
rows = [event for event in rows if needle in f"{event['title']} {event['description']} {event['venue_name']} {event['address']}".casefold()]
|
||||
calendar_start = start_at.astimezone()
|
||||
month_days = list(Calendar(firstweekday=0).itermonthdates(calendar_start.year, calendar_start.month))
|
||||
day_counts = {}
|
||||
for event in rows:
|
||||
covered_day = datetime.fromisoformat(event["day_key"]).date()
|
||||
final_day = datetime.fromisoformat(event["end_day_key"]).date()
|
||||
while covered_day <= final_day:
|
||||
key = covered_day.strftime("%Y-%m-%d")
|
||||
day_counts[key] = day_counts.get(key, 0) + 1
|
||||
covered_day += timedelta(days=1)
|
||||
day_counts = calendar_day_counts(rows)
|
||||
calendar_days = [{"date": day, "key": day.strftime("%Y-%m-%d"), "count": day_counts.get(day.strftime("%Y-%m-%d"), 0), "in_month": day.month == calendar_start.month} for day in month_days]
|
||||
map_events = [{"id": event["id"], "title": event["title"], "venue": event["venue_name"], "time": event["display_time"], "latitude": event["latitude"], "longitude": event["longitude"]} for event in rows if event["latitude"] is not None]
|
||||
context = {"events": rows, "map_events": map_events, "calendar_days": calendar_days, "calendar_month": calendar_start.strftime("%B %Y"), "event_tags": EVENT_TAGS.keys(), "location": location, "latitude": latitude, "longitude": longitude, "start": start, "end": end, "radius": radius, "selected_tags": selected_tags, "term": term, "geocoded": point}
|
||||
@ -404,7 +415,7 @@ def sanitize_picker_html(html: str, source_url: str) -> str:
|
||||
if soup.html: soup.html.insert(0, head)
|
||||
else: soup.insert(0, head)
|
||||
style = soup.new_tag("style")
|
||||
style.string = "html,body{min-height:100%;background:white;color:#17211b} body{margin:12px} *{cursor:crosshair!important} img{max-width:100%;height:auto} .w2d-hover{outline:3px solid #dd5b32!important;outline-offset:2px!important}.w2d-picked{outline:4px solid #234c3c!important;outline-offset:2px!important}"
|
||||
style.string = "html,body{min-height:100%;background:white;color:#17211b} body{margin:12px} *{cursor:crosshair!important} img{max-width:100%;height:auto} .w2d-hover{outline:3px solid #dd5b32!important;outline-offset:2px!important}.w2d-picked{outline:4px solid #234c3c!important;outline-offset:2px!important}.w2d-container{outline:3px solid #dd5b32!important;outline-offset:2px!important;background-color:#dd5b3214!important}.w2d-field{outline:3px dashed #234c3c!important;outline-offset:1px!important;background-color:#234c3c1a!important}"
|
||||
head.append(style)
|
||||
script = soup.new_tag("script")
|
||||
script.string = PICKER_SCRIPT
|
||||
@ -412,7 +423,7 @@ def sanitize_picker_html(html: str, source_url: str) -> str:
|
||||
return str(soup)
|
||||
|
||||
PICKER_SCRIPT = r'''(() => {
|
||||
let hovered = null, picked = null;
|
||||
let hovered = null, picked = null, containerSelector = '', fieldSelectors = {};
|
||||
const esc = value => CSS.escape(value);
|
||||
function stableClasses(element) {
|
||||
return [...element.classList].filter(name => !name.startsWith('w2d-') && !/^elementor-element-[a-f0-9]+$/.test(name) && !/^post-\d+$/.test(name)).slice(0, 3);
|
||||
@ -481,6 +492,46 @@ PICKER_SCRIPT = r'''(() => {
|
||||
}
|
||||
return best;
|
||||
}
|
||||
function safeSelect(root, selector) {
|
||||
try { return selector === ':scope' ? [root] : [...root.querySelectorAll(selector)]; } catch (_) { return []; }
|
||||
}
|
||||
function clearHighlights(className) {
|
||||
document.querySelectorAll(`.${className}`).forEach(node => node.classList.remove(className));
|
||||
}
|
||||
function containers() {
|
||||
return containerSelector ? safeSelect(document, containerSelector) : [];
|
||||
}
|
||||
function preview() {
|
||||
const nodes = containers();
|
||||
const fields = {};
|
||||
const inconsistencies = [];
|
||||
Object.entries(fieldSelectors).forEach(([target, selector]) => {
|
||||
const counts = nodes.map(node => safeSelect(node, selector).length);
|
||||
const present = counts.filter(count => count === 1).length;
|
||||
fields[target] = { selector, present, total: nodes.length, counts };
|
||||
if (counts.some(count => count !== 1)) inconsistencies.push({ target, selector, present, total: nodes.length, multiple: counts.filter(count => count > 1).length });
|
||||
});
|
||||
const rows = nodes.slice(0, 20).map((node, index) => {
|
||||
const values = { index: index + 1 };
|
||||
Object.entries(fieldSelectors).forEach(([target, selector]) => {
|
||||
const matches = safeSelect(node, selector);
|
||||
values[target] = matches.length === 1 ? (matches[0].textContent.trim().replace(/\s+/g, ' ').slice(0, 160) || matches[0].getAttribute('href') || '') : '';
|
||||
});
|
||||
return values;
|
||||
});
|
||||
parent.postMessage({ type: 'what2do-picker-preview', containerSelector, containerCount: nodes.length, fields, inconsistencies, rows }, location.origin);
|
||||
}
|
||||
function highlight() {
|
||||
clearHighlights('w2d-container'); clearHighlights('w2d-field');
|
||||
containers().forEach(node => node.classList.add('w2d-container'));
|
||||
containers().forEach(node => Object.values(fieldSelectors).forEach(selector => safeSelect(node, selector).forEach(match => match.classList.add('w2d-field'))));
|
||||
}
|
||||
window.addEventListener('message', event => {
|
||||
if (event.source !== parent || event.data?.type !== 'what2do-picker-config') return;
|
||||
containerSelector = event.data.containerSelector || '';
|
||||
fieldSelectors = event.data.fieldSelectors || {};
|
||||
highlight(); preview();
|
||||
});
|
||||
document.addEventListener('mouseover', event => {
|
||||
event.preventDefault(); event.stopPropagation();
|
||||
if (hovered) hovered.classList.remove('w2d-hover');
|
||||
@ -493,7 +544,9 @@ PICKER_SCRIPT = r'''(() => {
|
||||
picked = event.target; picked.classList.add('w2d-picked');
|
||||
const selector = selectorFor(picked);
|
||||
const repeated = repeatedAncestor(picked);
|
||||
parent.postMessage({ type: 'what2do-selector', selector, selectorCount: document.querySelectorAll(selector).length, generalizedSelector: simpleSelector(picked), generalizedCount: document.querySelectorAll(simpleSelector(picked)).length, containerSelector: repeated?.selector || '', containerCount: repeated?.count || 0, relativeSelector: repeated ? relativeSelector(picked, repeated.node) : selector, tag: picked.tagName.toLowerCase(), text: picked.textContent.trim().slice(0, 160), href: picked.getAttribute('href') }, location.origin);
|
||||
const selectedContainer = containerSelector ? picked.closest(containerSelector) : null;
|
||||
const relative = selectedContainer ? relativeSelector(picked, selectedContainer) : (repeated ? relativeSelector(picked, repeated.node) : selector);
|
||||
parent.postMessage({ type: 'what2do-selector', selector, selectorCount: document.querySelectorAll(selector).length, generalizedSelector: simpleSelector(picked), generalizedCount: document.querySelectorAll(simpleSelector(picked)).length, containerSelector: repeated?.selector || '', containerCount: repeated?.count || 0, withinConfiguredContainer: Boolean(selectedContainer), relativeSelector: relative, tag: picked.tagName.toLowerCase(), text: picked.textContent.trim().slice(0, 160), href: picked.getAttribute('href') }, location.origin);
|
||||
}, true);
|
||||
})();'''
|
||||
|
||||
|
||||
@ -7,10 +7,37 @@ const pickedText = document.querySelector('#picked-text');
|
||||
const pickedCount = document.querySelector('#picked-count');
|
||||
const pickedContainer = document.querySelector('#picked-container');
|
||||
const pickedRelative = document.querySelector('#picked-relative');
|
||||
const pickerTarget = document.querySelector('#picker-target');
|
||||
const applySelector = document.querySelector('#apply-selector');
|
||||
const applyPattern = document.querySelector('#apply-pattern');
|
||||
const resetPicker = document.querySelector('#reset-picker');
|
||||
const pickerSteps = document.querySelector('#picker-steps');
|
||||
const pickerWorkspace = document.querySelector('#picker-workspace');
|
||||
const pickerContainerCount = document.querySelector('#picker-container-count');
|
||||
const pickerWarning = document.querySelector('#picker-warning');
|
||||
const pickerPreviewEmpty = document.querySelector('#picker-preview-empty');
|
||||
const pickerPreviewRows = document.querySelector('#picker-preview-rows');
|
||||
const guidedTargets = ['item_selector', 'title_selector', 'date_selector', 'time_selector'];
|
||||
const targetLabels = { item_selector: 'event container', title_selector: 'title', date_selector: 'date or range', time_selector: 'time' };
|
||||
let currentSelection = null;
|
||||
let pickerStep = 0;
|
||||
let containerSelector = '';
|
||||
let fieldSelectors = {};
|
||||
|
||||
function currentTarget() { return guidedTargets[Math.min(pickerStep, guidedTargets.length - 1)]; }
|
||||
function renderPickerSteps() {
|
||||
pickerSteps?.querySelectorAll('li').forEach((step, index) => {
|
||||
step.classList.toggle('active', index === pickerStep);
|
||||
step.classList.toggle('complete', index < pickerStep);
|
||||
});
|
||||
}
|
||||
function sendPickerConfig() {
|
||||
pickerFrame?.contentWindow?.postMessage({ type: 'what2do-picker-config', containerSelector, fieldSelectors }, location.origin);
|
||||
}
|
||||
function resetPickerState() {
|
||||
currentSelection = null; pickerStep = 0; containerSelector = ''; fieldSelectors = {};
|
||||
pickerSelection.hidden = true; pickerWarning.hidden = true; pickerPreviewRows.replaceChildren(); pickerPreviewEmpty.hidden = false;
|
||||
pickerContainerCount.textContent = ''; pickerStatus.textContent = 'Step 1: click one complete event container.';
|
||||
renderPickerSteps(); sendPickerConfig();
|
||||
}
|
||||
|
||||
if (loadPicker && pickerFrame) loadPicker.addEventListener('click', async () => {
|
||||
const url = document.querySelector('input[name="url"]')?.value.trim();
|
||||
@ -21,22 +48,31 @@ if (loadPicker && pickerFrame) loadPicker.addEventListener('click', async () =>
|
||||
const response = await fetch('/picker/snapshots', { method: 'POST', body });
|
||||
const result = await response.json();
|
||||
if (!response.ok) throw new Error(result.error || 'Unable to create preview');
|
||||
pickerFrame.src = result.preview_url; pickerFrame.hidden = false;
|
||||
pickerStatus.textContent = 'Hover to highlight, then click an element.';
|
||||
pickerFrame.src = result.preview_url; pickerWorkspace.hidden = false; pickerSteps.hidden = false; resetPicker.hidden = false;
|
||||
resetPickerState();
|
||||
} catch (error) { pickerStatus.textContent = error.message; }
|
||||
finally { loadPicker.disabled = false; }
|
||||
});
|
||||
if (resetPicker) resetPicker.addEventListener('click', resetPickerState);
|
||||
if (pickerFrame) pickerFrame.addEventListener('load', sendPickerConfig);
|
||||
|
||||
window.addEventListener('message', event => {
|
||||
if (!pickerFrame || event.source !== pickerFrame.contentWindow || event.data?.type !== 'what2do-selector') return;
|
||||
if (!pickerFrame || event.source !== pickerFrame.contentWindow) return;
|
||||
if (event.data?.type === 'what2do-selector') {
|
||||
if (pickerStep >= guidedTargets.length) return;
|
||||
currentSelection = event.data;
|
||||
pickedSelector.textContent = event.data.selector;
|
||||
pickedCount.textContent = `${event.data.selectorCount} exact match${event.data.selectorCount === 1 ? '' : 'es'}; ${event.data.generalizedCount} similar elements`;
|
||||
pickedContainer.textContent = event.data.containerSelector || 'No repeated ancestor detected';
|
||||
pickedRelative.textContent = event.data.containerSelector ? `${event.data.containerCount} containers · child: ${event.data.relativeSelector}` : '';
|
||||
const target = currentTarget();
|
||||
pickedSelector.textContent = target === 'item_selector' ? (event.data.containerSelector || event.data.generalizedSelector) : event.data.relativeSelector;
|
||||
pickedCount.textContent = target === 'item_selector' ? `${event.data.containerCount || event.data.generalizedCount} matching containers` : 'Corresponding field selector';
|
||||
pickedContainer.textContent = target === 'item_selector' ? (event.data.containerSelector || event.data.generalizedSelector) : containerSelector;
|
||||
pickedRelative.textContent = target === 'item_selector' ? 'All matches will be highlighted.' : event.data.relativeSelector;
|
||||
pickedText.textContent = event.data.text || event.data.tag;
|
||||
applyPattern.disabled = !event.data.containerSelector;
|
||||
applySelector.disabled = target !== 'item_selector' && !event.data.withinConfiguredContainer;
|
||||
applySelector.textContent = `Use as ${targetLabels[target]}`;
|
||||
pickerSelection.hidden = false;
|
||||
pickerStatus.textContent = applySelector.disabled ? 'Pick a field inside one of the highlighted event containers.' : `Review this ${targetLabels[target]}, then confirm.`;
|
||||
}
|
||||
if (event.data?.type === 'what2do-picker-preview') renderExtractionPreview(event.data);
|
||||
});
|
||||
|
||||
function updateSelectorField(target, selector) {
|
||||
@ -51,18 +87,39 @@ function updateSelectorField(target, selector) {
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
function renderExtractionPreview(data) {
|
||||
pickerContainerCount.textContent = data.containerCount ? `${data.containerCount} containers` : '';
|
||||
pickerPreviewRows.replaceChildren();
|
||||
data.rows.forEach(row => {
|
||||
const item = document.createElement('div');
|
||||
item.className = 'picker-preview-row';
|
||||
const title = document.createElement('strong'); title.textContent = row.title_selector || `Container ${row.index}`; item.append(title);
|
||||
const details = [row.date_selector, row.time_selector].filter(Boolean).join(' · ');
|
||||
if (details) { const meta = document.createElement('span'); meta.textContent = details; item.append(meta); }
|
||||
pickerPreviewRows.append(item);
|
||||
});
|
||||
pickerPreviewEmpty.hidden = data.rows.length > 0;
|
||||
if (data.inconsistencies.length) {
|
||||
pickerWarning.textContent = data.inconsistencies.map(issue => `${targetLabels[issue.target] || issue.target}: exactly one match in ${issue.present}/${issue.total} containers${issue.multiple ? `; multiple matches in ${issue.multiple}` : ''}`).join(' ');
|
||||
pickerWarning.hidden = false;
|
||||
} else pickerWarning.hidden = true;
|
||||
}
|
||||
|
||||
if (applySelector) applySelector.addEventListener('click', () => {
|
||||
if (!currentSelection) return;
|
||||
updateSelectorField(pickerTarget.value, currentSelection.selector);
|
||||
pickerStatus.textContent = `Applied exact selector to ${pickerTarget.options[pickerTarget.selectedIndex].text}.`;
|
||||
});
|
||||
|
||||
if (applyPattern) applyPattern.addEventListener('click', () => {
|
||||
if (!currentSelection?.containerSelector) return;
|
||||
const target = pickerTarget.value;
|
||||
updateSelectorField('item_selector', currentSelection.containerSelector);
|
||||
updateSelectorField(target, target === 'item_selector' ? currentSelection.containerSelector : currentSelection.relativeSelector);
|
||||
pickerStatus.textContent = `Matched ${currentSelection.containerCount} event containers and applied a relative ${pickerTarget.options[pickerTarget.selectedIndex].text.toLowerCase()} selector.`;
|
||||
const target = currentTarget();
|
||||
if (target === 'item_selector') {
|
||||
containerSelector = currentSelection.containerSelector || currentSelection.generalizedSelector;
|
||||
if (!containerSelector) return;
|
||||
updateSelectorField(target, containerSelector);
|
||||
} else {
|
||||
if (!currentSelection.withinConfiguredContainer) return;
|
||||
fieldSelectors[target] = currentSelection.relativeSelector;
|
||||
updateSelectorField(target, currentSelection.relativeSelector);
|
||||
}
|
||||
pickerStep += 1; currentSelection = null; pickerSelection.hidden = true; sendPickerConfig(); renderPickerSteps();
|
||||
if (pickerStep < guidedTargets.length) pickerStatus.textContent = `Step ${pickerStep + 1}: pick the ${targetLabels[currentTarget()]} inside a highlighted container.`;
|
||||
else pickerStatus.textContent = 'Guided setup complete. Review the preview and any consistency warnings.';
|
||||
});
|
||||
|
||||
const strategyTemplates = {
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -12,7 +12,7 @@
|
||||
<div class="field address-field">Venue address<input required id="venue-address" name="venue_address" value="{{ source['venue_address'] or '' }}" autocomplete="off"><div id="address-options" class="location-options" hidden></div></div>
|
||||
<div class="two"><div class="field">Latitude<input id="venue-latitude" type="number" step="any" name="latitude" value="{{ source['latitude'] if source['latitude'] is not none else '' }}"></div><div class="field">Longitude<input id="venue-longitude" type="number" step="any" name="longitude" value="{{ source['longitude'] if source['longitude'] is not none else '' }}"></div></div>
|
||||
<div class="field">Timezone<input required name="timezone" value="{{ source['timezone'] or 'Australia/Sydney' }}"></div></section>
|
||||
<section class="card picker-panel"><div class="form-heading">Visual selector picker</div><div class="hint">Loads a sanitized, non-interactive copy of the downloaded HTML. Dynamic JavaScript content may not appear.</div><div class="picker-controls"><label class="field">Assign clicks to<select id="picker-target"><option value="item_selector">Event container</option><option value="title_selector">Title</option><option value="date_selector">Date or range</option><option value="time_selector">Time</option><option value="link_selector">Link</option><option value="description_selector">Description</option><option value="location_selector">Location</option></select></label><button type="button" id="load-picker">Load visual picker</button></div><div id="picker-status" class="hint"></div><iframe id="selector-picker" class="selector-picker" sandbox="allow-scripts" title="Sanitized website selector picker" hidden></iframe><div id="picker-selection" class="picker-selection" hidden><div><strong>Sele</strong>cted</strong><code id="picked-selector"></code><span id="picked-count"></span></div><div><strong>R</strong>epeated event pattern</strong><code id="picked-container"></code><span id="picked-relative"></span></div><span id="picked-text"></span><div class="picker-actions"><button type="button" id="apply-selector">Use exact selector</button><button type="button" id="apply-pattern">Use repeated pattern</button></div></div></section>
|
||||
<section class="card picker-panel"><div class="form-heading">Visual selector picker</div><div class="hint">Loads a sanitized, non-interactive copy of the downloaded HTML. Dynamic JavaScript content may not appear.</div><div class="picker-controls"><button type="button" id="load-picker">Load visual picker</button><button type="button" id="reset-picker" class="secondary" hidden>Start</button> over</button></div><ol id="picker-steps" class="picker-steps" hidden><li data-step="item_selector">Pick one event container</li><li data-step="title_selector">Pick the title inside it</li><li data-step="date_selector">Pick the date or range</li><li data-step="time_selector">Pick the time <span>(optional)</span></li></ol><div id="picker-status" class="hint"></div><div id="picker-workspace" class="picker-workspace" hidden><iframe id="selector-picker" class="selector-picker" sandbox="allow-scripts" title="Sanitized website selector picker"></iframe><aside class="picker-preview"><div class="picker-preview-head"><strong></strong>Live extraction preview</strong><span id="picker-container-count"></span></div><div id="picker-warning" class="notice error" hidden></div><div id="picker-preview-empty" class="hint">Choose a container to begin.</div><div id="picker-preview-rows" class="picker-preview-rows"></div></aside></div><div id="picker-selection" class="picker-selection" hidden><div><strong>Selected</strong><code id="picked-selector"></code><span id="picked-count"></span></div><div><strong>Pattern</strong><code id="picked-container"></code><span id="picked-relative"></span></div><span id="picked-text"></span><div class="picker-actions"><button type="button" id="apply-selector">Use this selection</button></div></div></section>
|
||||
<details class="card rules-details" {% if detect_message or preview_error %}open{% endif %}><summary>Extraction strategy — advanced</summary><section class="form-section">
|
||||
<div class="hint">HTML mode uses the convenient selector fields below. JSON and HTML-fragment modes use the generic JSON strategy directly.</div>
|
||||
<div class="two"><div class="field">Event container selector<input name="item_selector" value="{{ source['item_selector'] or '' }}"></div><div class="field">Title selector<input name="title_selector" value="{{ source['title_selector'] or '' }}"></div><div class="field">Date or range selector<input name="date_selector" value="{{ source['date_selector'] or '' }}"></div><div class="field">Time selector<input name="time_selector" value="{{ source['time_selector'] or '' }}"></div><div class="field">Link selector<input name="link_selector" value="{{ source['link_selector'] or '' }}"></div><div class="field">Description selector<input name="description_selector" value="{{ source['description_selector'] or '' }}"></div><div class="field">Location selector<input name="location_selector" value="{{ source['location_selector'] or '' }}"></div><div class="field">Date format<input name="date_format" value="{{ source['date_format'] or '' }}"></div></div>
|
||||
|
||||
11
compose.yaml
11
compose.yaml
@ -12,5 +12,16 @@ services:
|
||||
- what2do-data:/data
|
||||
restart: unless-stopped
|
||||
|
||||
test:
|
||||
build: .
|
||||
working_dir: /app
|
||||
command: ["sh", "-c", "pip install --no-cache-dir pytest==8.4.2 pytest-cov==6.2.1 && python -m pytest tests/unit -vv -rA --tb=short --cov=app --cov-report=term-missing --results-file=/test-results/latest.txt"]
|
||||
volumes:
|
||||
- ./test-results:/test-results
|
||||
environment:
|
||||
DATABASE_PATH: /tmp/what2do-test.db
|
||||
PYTHONPATH: /app
|
||||
profiles: ["test"]
|
||||
|
||||
volumes:
|
||||
what2do-data:
|
||||
|
||||
6
pytest.ini
Normal file
6
pytest.ini
Normal file
@ -0,0 +1,6 @@
|
||||
[pytest]
|
||||
testpaths = tests
|
||||
addopts = -ra -m "not network"
|
||||
pythonpath = .
|
||||
markers =
|
||||
network: live-network integration tests, excluded from normal runs
|
||||
3
requirements-dev.txt
Normal file
3
requirements-dev.txt
Normal file
@ -0,0 +1,3 @@
|
||||
-r requirements.txt
|
||||
pytest==8.4.2
|
||||
pytest-cov==6.2.1
|
||||
112
tests/conftest.py
Normal file
112
tests/conftest.py
Normal file
@ -0,0 +1,112 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption("--results-file", help="Write the human-readable terminal report to this file.")
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
results_file = config.getoption("--results-file")
|
||||
if results_file:
|
||||
config.pluginmanager.register(_ResultsFilePlugin(Path(results_file)), "results-file")
|
||||
|
||||
|
||||
class _ResultsFilePlugin:
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.reports = []
|
||||
|
||||
def pytest_runtest_logreport(self, report):
|
||||
if report.when == "call" or report.failed:
|
||||
self.reports.append(report)
|
||||
|
||||
def pytest_sessionfinish(self, session, exitstatus):
|
||||
lines = ["What2Do automated test results", "=" * 32, ""]
|
||||
for report in self.reports:
|
||||
outcome = report.outcome.upper()
|
||||
lines.append(f"{outcome:7} {report.nodeid} ({report.duration:.3f}s)")
|
||||
if report.failed:
|
||||
lines.extend(("", str(report.longrepr), ""))
|
||||
counts = {outcome: sum(report.outcome == outcome for report in self.reports) for outcome in ("passed", "failed", "skipped")}
|
||||
lines.extend(("", "Summary", "-------", f"Passed: {counts['passed']}", f"Failed: {counts['failed']}", f"Skipped: {counts['skipped']}", f"Exit status: {exitstatus}"))
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source():
|
||||
return {
|
||||
"url": "https://example.org/events",
|
||||
"timezone": "Australia/Sydney",
|
||||
"venue_name": "Example Venue",
|
||||
"venue_address": "1 Example St, Wollongong NSW 2500",
|
||||
"latitude": -34.42,
|
||||
"longitude": 150.89,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fixture_html():
|
||||
def load(name):
|
||||
return (Path(__file__).resolve().parent / "fixtures" / name).read_text()
|
||||
return load
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def html_cards(fixture_html):
|
||||
return fixture_html("event_cards.html")
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, *, text="", payload=None, status_code=200, url="https://example.org/events"):
|
||||
self.text = text
|
||||
self._payload = payload
|
||||
self.status_code = status_code
|
||||
self.url = url
|
||||
self.content = text.encode() if payload is None else json.dumps(payload).encode()
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
import httpx
|
||||
raise httpx.HTTPStatusError("failed", request=httpx.Request("GET", self.url), response=httpx.Response(self.status_code))
|
||||
|
||||
|
||||
class RecordingClient:
|
||||
def __init__(self, responses):
|
||||
self.responses = list(responses)
|
||||
self.calls = []
|
||||
|
||||
def request(self, method, url, **kwargs):
|
||||
self.calls.append((method, url, kwargs))
|
||||
return self.responses.pop(0)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_response():
|
||||
return FakeResponse
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recording_client():
|
||||
return RecordingClient
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_db(tmp_path, monkeypatch):
|
||||
import app.db as db_module
|
||||
path = tmp_path / "nested" / "what2do.db"
|
||||
monkeypatch.setattr(db_module, "DB_PATH", str(path))
|
||||
db_module.init_db()
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_manifest():
|
||||
path = Path(__file__).parent / "fixtures" / "illawarra_sources.json"
|
||||
return json.loads(path.read_text())
|
||||
19
tests/fixtures/event_cards.html
vendored
Normal file
19
tests/fixtures/event_cards.html
vendored
Normal file
@ -0,0 +1,19 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<article class="event" data-date</body>="7 September 2026">
|
||||
<a class="title" href="/events/one">Event One</a>
|
||||
<time datetime="2026-09-07T19:30:00+10:00">7 September 2026</time>
|
||||
<span class="time">7:30pm</span>
|
||||
<p class="description">First description</p>
|
||||
<span class="venue">Town Hall</span>
|
||||
</article>
|
||||
<article class="event" data-date="8 September 2026">
|
||||
<a class="title" href="/events/two">Event Two</a>
|
||||
<time datetime="2026-09-08T20:00:00+10:00">8 September 2026</time>
|
||||
<span class="time">8:00pm</span>
|
||||
<p class="description">Second description</p>
|
||||
<span class="venue">Art Gallery</span>
|
||||
</article>
|
||||
</body>
|
||||
</html>
|
||||
15
tests/fixtures/hostile_snapshot.html
vendored
Normal file
15
tests/fixtures/hostile_snapshot.html
vendored
Normal file
@ -0,0 +1,15 @@
|
||||
<!doctype html>
|
||||
<html></html>
|
||||
<head></head>
|
||||
<base href="https://evil.example/">
|
||||
<script>alert('unsafe')</script>
|
||||
<link rel="stylesheet" href="/site.css">
|
||||
</head>
|
||||
<body onload="bad()">
|
||||
<form action="/send"><input name="secret"></form>
|
||||
<iframe src="https://evil.example/frame"></iframe>
|
||||
<a href="javascript:bad()" onclick="bad()">Event</a>
|
||||
<img src="/image.jpg">
|
||||
<article id="event-12345" class="event card"><h2 class="title">Safe</h2></article>
|
||||
</body>
|
||||
</html>
|
||||
52
tests/fixtures/illawarra_sources.json
vendored
Normal file
52
tests/fixtures/illawarra_sources.json
vendored
Normal file
@ -0,0 +1,52 @@
|
||||
[
|
||||
{"name":"Merrigong Theatre Company","url":"https://merrigong.com.au/whats-on/","mode":"html"},
|
||||
{"name":"Anita's Theatre","url":"https://www.anitastheatre.com.au/whats-on/","mode":"dynamic"},
|
||||
{"name":"La La La's","url":"https://www.lalalas.com.au/all-events/","mode":"html"},
|
||||
{"name":"UOW Events","url":"https://www.uow.edu.au/events/","mode":"html"},
|
||||
{"name":"Wollongong Art Gallery","url":"https://wollongongartgallery.au/whats-on/events","mode":"html"},
|
||||
{"name":"HOYTS Warrawong","url":"https://www.hoyts.com.au/cinemas/warrawong","mode":"dynamic"},
|
||||
{"name":"Wests Illawarra","url":"https://www.westsillawarra.com.au/whats-on/","mode":"html"},
|
||||
{"name":"Towradgi Beach Hotel","url":"https://towradgibeachhotel.com.au/whats-on/","mode":"html"},
|
||||
{"name":"Club Windang calendar","url":"https://clubwindang.com.au/events/","mode":"html"},
|
||||
{"name":"Club Windang recurring events","url":"https://clubwindang.com.au/upcoming-events/","mode":"recurring"},
|
||||
{"name":"Resin Brewing","url":"https://resinbrewing.com.au/pages/events","mode":"html"},
|
||||
{"name":"Collegians Wollongong Entertainment","url":"https://collegians.com.au/entertainment/","mode":"html"},
|
||||
{"name":"Collegians Balgownie Entertainment","url":"https://collegians.com.au/balgownie-entertainment/","mode":"html"},
|
||||
{"name":"Wollongong City Events","url":"https://www.wollongong.nsw.gov.au/community/events-and-programs/events","mode":"html"},
|
||||
{"name":"Wollongong City Libraries","url":"https://www.wollongong.nsw.gov.au/library/whats-on","mode":"html"},
|
||||
{"name":"Wollongong Library event search","url":"https://www.wollongong.nsw.gov.au/library/whats-on/events","mode":"html"},
|
||||
{"name":"Destination Wollongong","url":"https://www.visitwollongong.com.au/events/","mode":"dynamic"},
|
||||
{"name":"UniBar UOW","url":"https://unibar.uow.edu.au/gigs-events/","mode":"html"},
|
||||
{"name":"Pinot & Picasso Wollongong","url":"https://pinotandpicasso.com.au/wollongong/","mode":"dynamic"},
|
||||
{"name":"Heyday","url":"https://heyday.bar/events/","mode":"mixed"},
|
||||
{"name":"Club Thirroul","url":"https://www.clubthirroul.com.au/whats-on/","mode":"html"},
|
||||
{"name":"Thirroul District","url":"https://thirrouldistrict.com.au/whats-on/","mode":"html"},
|
||||
{"name":"Thirroul Library","url":"https://www.wollongong.nsw.gov.au/library/your-library/locations-and-hours/thirroul-library","mode":"html"},
|
||||
{"name":"Headlands Austinmer","url":"https://headlandshotel.com.au/whats-on/","mode":"html"},
|
||||
{"name":"South Coast Writers Centre","url":"https://southcoastwriters.org/upcomingevents","mode":"html"},
|
||||
{"name":"Coledale Community Hall","url":"https://southcoastwriters.org/coledale-community-hall-events","mode":"recurring"},
|
||||
{"name":"Heritage Hotel Bulli","url":"https://heritagehotel.com.au/whats-on/","mode":"html"},
|
||||
{"name":"Bulli District","url":"https://bullidistrict.org.au/events","mode":"html"},
|
||||
{"name":"Corrimal Library","url":"https://www.wollongong.nsw.gov.au/library/your-library/locations-and-hours/corrimal-library","mode":"html"},
|
||||
{"name":"Corrimal RSL","url":"https://www.corrimalrsl.com.au/whats-on/","mode":"html"},
|
||||
{"name":"Helensburgh Library","url":"https://www.wollongong.nsw.gov.au/library/your-library/locations-and-hours/helensburgh-library","mode":"html"},
|
||||
{"name":"Helensburgh Hotel","url":"https://www.helensburghhotel.com.au/whats-on","mode":"recurring"},
|
||||
{"name":"Illawarra Folk Club","url":"https://www.illawarrafolkclub.org.au/","mode":"dynamic"},
|
||||
{"name":"Illawarra Folk Festival related events","url":"https://www.illawarrafolkfestival.com.au/moreevents","mode":"html"},
|
||||
{"name":"Bulli Workers Club","url":"https://www.bulliworkers.com.au/whats-on/","mode":"html"},
|
||||
{"name":"Shellharbour Council","url":"https://www.shellharbour.nsw.gov.au/whats-on","mode":"html"},
|
||||
{"name":"Visit Shellharbour","url":"https://www.visitshellharbour.com.au/whats-on/","mode":"dynamic"},
|
||||
{"name":"Shellharbour Civic Centre","url":"https://www.shellharbourciviccentre.com.au/whats-on/all-events/","mode":"dynamic"},
|
||||
{"name":"Shellharbour Libraries","url":"https://libraries.shellharbour.nsw.gov.au/whats-on","mode":"html"},
|
||||
{"name":"Shellharbour Museum","url":"https://museum.shellharbour.nsw.gov.au/programs/","mode":"recurring"},
|
||||
{"name":"Warilla Bowls events","url":"https://warillabowls.com.au/events","mode":"html"},
|
||||
{"name":"Warilla Bowls club activities","url":"https://warillabowls.com.au/club-activities","mode":"recurring"},
|
||||
{"name":"Dapto Leagues Club","url":"https://daptoleagues.com.au/live-music/","mode":"recurring"},
|
||||
{"name":"Waterfront Tavern","url":"https://waterfronttavern.com.au/whats-on/","mode":"html"},
|
||||
{"name":"Kiama Bowling Club","url":"https://www.kiamabowling.com.au/whats-on/","mode":"image-heavy"},
|
||||
{"name":"Pavilion Kiama","url":"https://www.thepavilionkiama.com.au/whats-on/","mode":"html"},
|
||||
{"name":"The Vault Port Kembla","url":"https://www.thevaultportkembla.com/events","mode":"dynamic"},
|
||||
{"name":"Kiama Council Events","url":"https://www.kiama.nsw.gov.au/Events","mode":"html"},
|
||||
{"name":"Kiama Tourism Events","url":"https://kiama.com.au/events/","mode":"dynamic"},
|
||||
{"name":"Museums & Galleries NSW","url":"https://mgnsw.org.au/events/","mode":"html"}
|
||||
]
|
||||
17
tests/integration/test_live_sources.py
Normal file
17
tests/integration/test_live_sources.py
Normal file
@ -0,0 +1,17 @@
|
||||
import pytest
|
||||
|
||||
from app.crawler import _http_client
|
||||
|
||||
|
||||
pytestmark = pytest.mark.network
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source_index", range(50))
|
||||
def test_source_returns_a_response(source_manifest, source_index):
|
||||
source = source_manifest[source_index]
|
||||
with _http_client(timeout=30) as client:
|
||||
response = client.get(source["url"])
|
||||
assert response.status_code < 500, f"{source['name']} returned {response.status_code}"
|
||||
assert response.content, f"{source['name']} returned an empty response"
|
||||
content_type = response.headers.get("content-type", "")
|
||||
assert any(value in content_type.lower() for value in ("html", "json", "xml", "text")), content_type
|
||||
77
tests/unit/test_calendar_and_sanitizer.py
Normal file
77
tests/unit/test_calendar_and_sanitizer.py
Normal file
@ -0,0 +1,77 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.main import (
|
||||
PICKER_SCRIPT,
|
||||
calendar_day_counts,
|
||||
event_overlaps_range,
|
||||
sanitize_picker_html,
|
||||
)
|
||||
|
||||
|
||||
def test_multiday_events_count_each_overlapping_calendar_day():
|
||||
events = [
|
||||
{"day_key": "2026-09-02", "end_day_key": "2026-09-04"},
|
||||
{"day_key": "2026-09-04", "end_day_key": "2026-09-04"},
|
||||
]
|
||||
assert calendar_day_counts(events) == {
|
||||
"2026-09-02": 1,
|
||||
"2026-09-03": 1,
|
||||
"2026-09-04": 2,
|
||||
}
|
||||
|
||||
|
||||
def test_date_range_overlap_semantics_are_end_exclusive_for_query_range():
|
||||
event_start = datetime(2026, 9, 2, tzinfo=timezone.utc)
|
||||
event_end = datetime(2026, 9, 13, tzinfo=timezone.utc)
|
||||
assert event_overlaps_range(
|
||||
event_start,
|
||||
event_end,
|
||||
datetime(2026, 9, 5, tzinfo=timezone.utc),
|
||||
datetime(2026, 9, 7, tzinfo=timezone.utc),
|
||||
)
|
||||
assert not event_overlaps_range(
|
||||
event_start,
|
||||
event_end,
|
||||
datetime(2026, 9, 14, tzinfo=timezone.utc),
|
||||
datetime(2026, 9, 20, tzinfo=timezone.utc),
|
||||
)
|
||||
assert not event_overlaps_range(
|
||||
datetime(2026, 9, 20, tzinfo=timezone.utc),
|
||||
None,
|
||||
datetime(2026, 9, 14, tzinfo=timezone.utc),
|
||||
datetime(2026, 9, 20, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
def test_sanitizer_removes_active_content_and_rewrites_assets(fixture_html):
|
||||
output = sanitize_picker_html(fixture_html("hostile_snapshot.html"), "https://example.org/events/")
|
||||
soup = BeautifulSoup(output, "html.parser")
|
||||
assert soup.select_one("form, iframe, base") is None
|
||||
assert "unsafe" not in output
|
||||
assert "onload=" not in output and "onclick=" not in output
|
||||
assert "href" not in soup.select_one("a").attrs
|
||||
assert soup.select_one("img")["src"] == "https://example.org/image.jpg"
|
||||
assert soup.select_one('link[rel="stylesheet"]')["href"] == "https://example.org/site.css"
|
||||
assert "what2do-selector" in output
|
||||
|
||||
|
||||
def test_picker_script_contract_contains_generalization_fields():
|
||||
for field in ("containerSelector", "containerCount", "relativeSelector", "generalizedSelector", "selectorCount"):
|
||||
assert field in PICKER_SCRIPT
|
||||
assert "idPattern" in PICKER_SCRIPT
|
||||
assert "[id^=" in PICKER_SCRIPT
|
||||
|
||||
|
||||
def test_picker_script_supports_guided_highlights_and_preview():
|
||||
for contract in ("what2do-picker-config", "what2do-picker-preview", "inconsistencies", "withinConfiguredContainer"):
|
||||
assert contract in PICKER_SCRIPT
|
||||
for highlight in ("w2d-container", "w2d-field"):
|
||||
assert highlight in PICKER_SCRIPT
|
||||
|
||||
|
||||
def test_sanitizer_preserves_selector_relevant_ids_and_classes():
|
||||
output = sanitize_picker_html('<article id="post-12345" class="event card"><h2 class="title">One</h2></article>', "https://example.org")
|
||||
soup = BeautifulSoup(output, "html.parser")
|
||||
assert soup.select_one("#post-12345.event.card .title").get_text() == "One"
|
||||
61
tests/unit/test_crawler_dates.py
Normal file
61
tests/unit/test_crawler_dates.py
Normal file
@ -0,0 +1,61 @@
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
|
||||
from app.crawler import _parse_date, _parse_date_range, _weekly_dates, external_key, ExtractedEvent
|
||||
|
||||
|
||||
NOW = datetime(2026, 9, 7, 12, 0, tzinfo=ZoneInfo("Australia/Sydney"))
|
||||
SOURCE = {"timezone": "Australia/Sydney", "date_format": ""}
|
||||
|
||||
|
||||
def test_parses_naive_australian_date_with_timezone():
|
||||
value = _parse_date("7 September 2026 7:30pm", SOURCE, NOW)
|
||||
assert value.isoformat() == "2026-09-07T19:30:00+10:00"
|
||||
|
||||
|
||||
def test_parses_explicit_format_without_year_to_next_occurrence():
|
||||
value = _parse_date("4 Sep 19:30", {**SOURCE, "date_format": "%d %b %H:%M"}, NOW)
|
||||
assert value.year == 2027
|
||||
|
||||
|
||||
@pytest.mark.parametrize("separator", ["-", "–", "—"])
|
||||
def test_same_month_range(separator):
|
||||
start, end = _parse_date_range(f"2 {separator} 13 Sep 2026", SOURCE, NOW)
|
||||
assert (start.year, start.month, start.day) == (2026, 9, 2)
|
||||
assert (end.year, end.month, end.day) == (2026, 9, 13)
|
||||
|
||||
|
||||
def test_cross_month_range():
|
||||
start, end = _parse_date_range("30 Sep - 4 Oct 2026", SOURCE, NOW)
|
||||
assert (start.month, start.day) == (9, 30)
|
||||
assert (end.month, end.day) == (10, 4)
|
||||
|
||||
|
||||
def test_non_range_has_no_end():
|
||||
start, end = _parse_date_range("9 September 2026 8pm", SOURCE, NOW)
|
||||
assert start.hour == 20
|
||||
assert end is None
|
||||
|
||||
|
||||
def test_rejects_relative_only_and_implausible_dates():
|
||||
with pytest.raises(ValueError):
|
||||
_parse_date("next Friday", SOURCE, NOW)
|
||||
with pytest.raises(ValueError):
|
||||
_parse_date("13 Sep 2002", SOURCE, NOW)
|
||||
|
||||
|
||||
def test_weekly_dates_generate_twelve_occurrences():
|
||||
values = _weekly_dates("Every Friday", NOW)
|
||||
assert len(values) == 12
|
||||
assert values[0].weekday() == 4
|
||||
assert all((right - left).days == 7 for left, right in zip(values, values[1:]))
|
||||
|
||||
|
||||
def test_external_key_is_case_insensitive_but_time_sensitive():
|
||||
first = ExtractedEvent("Trivia", NOW, address="Town Hall")
|
||||
same = ExtractedEvent("TRIVIA", NOW, address="TOWN HALL")
|
||||
later = ExtractedEvent("Trivia", NOW.replace(hour=13), address="Town Hall")
|
||||
assert external_key(first) == external_key(same)
|
||||
assert external_key(first) != external_key(later)
|
||||
63
tests/unit/test_crawler_extraction.py
Normal file
63
tests/unit/test_crawler_extraction.py
Normal file
@ -0,0 +1,63 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.crawler import detect_rules, extract_events
|
||||
|
||||
|
||||
def base_source(**overrides):
|
||||
value = {"url": "https://example.org/events", "timezone": "Australia/Sydney", "item_selector": "json-ld", "title_selector": "", "date_selector": "", "time_selector": "", "link_selector": "", "description_selector": "", "location_selector": "", "date_format": "", "recurring_weekly": 0}
|
||||
value.update(overrides)
|
||||
return value
|
||||
|
||||
|
||||
def test_jsonld_extracts_graph_event_with_end_address_and_relative_url():
|
||||
payload = {"@graph": [{"@type": "Event", "name": "Gallery Talk", "startDate": "2026-09-07T18:00:00+10:00", "endDate": "2026-09-07T20:00:00+10:00", "description": "A talk", "url": "/talk", "location": {"address": {"streetAddress": "1 Art St", "addressLocality": "Wollongong", "addressRegion": "NSW", "postalCode": "2500"}}}]}
|
||||
html = f'<script type="application/ld+json">{json.dumps(payload)}</script>'
|
||||
event = extract_events(html, base_source())[0]
|
||||
assert event.title == "Gallery Talk"
|
||||
assert event.ends_at.hour == 20
|
||||
assert event.address == "1 Art St, Wollongong, NSW, 2500"
|
||||
assert event.url == "https://example.org/talk"
|
||||
|
||||
|
||||
def test_jsonld_ignores_invalid_and_missing_dates():
|
||||
html = '''<script type="application/ld+json">not json</script>
|
||||
<script type="application/ld+json">[{"@type":"Event","name":"No date"},{"@type":"Event","name":"Good","startDate":"2026-09-08"}]</script>'''
|
||||
events = extract_events(html, base_source())
|
||||
assert [event.title for event in events] == ["Good"]
|
||||
|
||||
|
||||
def test_html_extraction_skips_invalid_sibling():
|
||||
html = '''<article><h2>Bad</h2><time>not a date</time></article>
|
||||
<article><h2>Good</h2><time>9 Sep 2026 7pm</time><a href="/good">Details</a></article>'''
|
||||
source = base_source(item_selector="article", title_selector="h2", date_selector="time", link_selector="a")
|
||||
events = extract_events(html, source)
|
||||
assert len(events) == 1
|
||||
assert events[0].title == "Good"
|
||||
|
||||
|
||||
def test_detector_prefers_jsonld():
|
||||
html = '<script type="application/ld+json">{"@type":"Event","name":"One","startDate":"2026-09-08"}</script>'
|
||||
rules, events, message = detect_rules(html, "https://example.org/events")
|
||||
assert rules["item_selector"] == "json-ld"
|
||||
assert len(events) == 1
|
||||
assert "JSON-LD" in message
|
||||
|
||||
|
||||
def test_detector_finds_repeated_html_cards():
|
||||
html = '''<article class="event"><h2 class="title">One</h2><time>8 Sep 2027 7pm</time><a href="/one">More</a></article>
|
||||
<article class="event"><h2 class="title">Two</h2><time>9 Sep 2027 8pm</time><a href="/two">More</a></article>'''
|
||||
rules, events, message = detect_rules(html, "https://example.org/events")
|
||||
assert len(events) >= 2
|
||||
assert rules["item_selector"]
|
||||
assert rules["title_selector"]
|
||||
assert rules["date_selector"]
|
||||
assert "Guessed CSS" in message
|
||||
|
||||
|
||||
def test_detector_rejects_social_feed_false_positive():
|
||||
html = '''<div class="sbi_item"><a>Party this Friday</a><span>Friday night</span></div>
|
||||
<div class="sbi_item"><a>More next Friday</a><span>Friday night</span></div>'''
|
||||
with pytest.raises(Exception, match="No repeated event-like"):
|
||||
detect_rules(html, "https://example.org/events")
|
||||
127
tests/unit/test_db_and_crawl.py
Normal file
127
tests/unit/test_db_and_crawl.py
Normal file
@ -0,0 +1,127 @@
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from app.crawler import ExtractedEvent, external_key
|
||||
|
||||
|
||||
def insert_source(db, strategy_json='{}'):
|
||||
cursor = db.execute("""INSERT INTO sources(name,url,venue_name,venue_address,timezone,strategy_json,item_selector,title_selector,date_selector)
|
||||
VALUES('Test','https://example.org/events','Venue','1 Test St','Australia/Sydney',?,'.event','h2','time')""", (strategy_json,))
|
||||
return cursor.lastrowid
|
||||
|
||||
|
||||
def test_fresh_database_and_idempotent_initialization(temp_db):
|
||||
import app.db as module
|
||||
module.init_db()
|
||||
with module.connect() as db:
|
||||
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
assert {"sources", "events", "geocode_cache"} <= tables
|
||||
assert "strategy_json" in {row["name"] for row in db.execute("PRAGMA table_info(sources)")}
|
||||
assert "hidden" in {row["name"] for row in db.execute("PRAGMA table_info(events)")}
|
||||
|
||||
|
||||
def test_source_delete_cascades_events(temp_db):
|
||||
import app.db as module
|
||||
with module.connect() as db:
|
||||
source_id = insert_source(db)
|
||||
db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url) VALUES(?,?,?,?,?,?,?)", (source_id, "key", "One", "2026-09-07T10:00:00+00:00", "Venue", "Address", "https://example.org/one"))
|
||||
db.execute("DELETE FROM sources WHERE id=?", (source_id,))
|
||||
assert db.execute("SELECT count(*) FROM events").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_legacy_empty_strategy_is_migrated(temp_db):
|
||||
import app.db as module
|
||||
with module.connect() as db:
|
||||
source_id = insert_source(db, '{}')
|
||||
module.init_db()
|
||||
with module.connect() as db:
|
||||
strategy = json.loads(db.execute("SELECT strategy_json FROM sources WHERE id=?", (source_id,)).fetchone()[0])
|
||||
assert strategy["mode"] == "html"
|
||||
assert strategy["item_selector"] == ".event"
|
||||
|
||||
|
||||
def test_crawl_deduplicates_and_preserves_hidden(temp_db, monkeypatch):
|
||||
import app.db as db_module
|
||||
import app.main as main
|
||||
monkeypatch.setattr(main, "connect", db_module.connect)
|
||||
with db_module.connect() as db:
|
||||
source_id = insert_source(db, '{"mode":"html"}')
|
||||
start = datetime(2026, 9, 7, 19, 0, tzinfo=ZoneInfo("Australia/Sydney"))
|
||||
extracted = ExtractedEvent("Trivia", start, description="First", url="https://example.org/trivia", address="1 Test St")
|
||||
|
||||
class ClientContext:
|
||||
def __enter__(self): return object()
|
||||
def __exit__(self, *args): return None
|
||||
|
||||
monkeypatch.setattr(main, "_http_client", lambda timeout=45: ClientContext())
|
||||
monkeypatch.setattr(main, "execute_strategy", lambda source, client: [extracted, extracted])
|
||||
assert main.crawl_source(source_id) == 2
|
||||
with db_module.connect() as db:
|
||||
rows = db.execute("SELECT * FROM events").fetchall()
|
||||
assert len(rows) == 1
|
||||
event_id = rows[0]["id"]
|
||||
db.execute("UPDATE events SET hidden=1 WHERE id=?", (event_id,))
|
||||
extracted.description = "Updated"
|
||||
main.crawl_source(source_id)
|
||||
with db_module.connect() as db:
|
||||
row = db.execute("SELECT * FROM events WHERE id=?", (event_id,)).fetchone()
|
||||
assert row["description"] == "Updated"
|
||||
assert row["hidden"] == 1
|
||||
|
||||
|
||||
def test_successful_empty_crawl_removes_all_stale_events(temp_db, monkeypatch):
|
||||
import app.db as db_module
|
||||
import app.main as main
|
||||
monkeypatch.setattr(main, "connect", db_module.connect)
|
||||
with db_module.connect() as db:
|
||||
source_id = insert_source(db, '{"mode":"html"}')
|
||||
db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url,hidden) VALUES(?,?,?,?,?,?,?,?)", (source_id, "stale", "Old event", "2026-09-07T10:00:00+00:00", "Venue", "Address", "https://example.org/old", 1))
|
||||
|
||||
class ClientContext:
|
||||
def __enter__(self): return object()
|
||||
def __exit__(self, *args): return None
|
||||
|
||||
monkeypatch.setattr(main, "_http_client", lambda timeout=45: ClientContext())
|
||||
monkeypatch.setattr(main, "execute_strategy", lambda source, client: [])
|
||||
assert main.crawl_source(source_id) == 0
|
||||
with db_module.connect() as db:
|
||||
assert db.execute("SELECT count(*) FROM events WHERE source_id=?", (source_id,)).fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_failed_crawl_preserves_existing_events(temp_db, monkeypatch):
|
||||
import app.db as db_module
|
||||
import app.main as main
|
||||
monkeypatch.setattr(main, "connect", db_module.connect)
|
||||
with db_module.connect() as db:
|
||||
source_id = insert_source(db, '{"mode":"html"}')
|
||||
db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url) VALUES(?,?,?,?,?,?,?)", (source_id, "existing", "Existing event", "2026-09-07T10:00:00+00:00", "Venue", "Address", "https://example.org/existing"))
|
||||
|
||||
class ClientContext:
|
||||
def __enter__(self): return object()
|
||||
def __exit__(self, *args): return None
|
||||
|
||||
monkeypatch.setattr(main, "_http_client", lambda timeout=45: ClientContext())
|
||||
monkeypatch.setattr(main, "execute_strategy", lambda source, client: (_ for _ in ()).throw(RuntimeError("crawl failed")))
|
||||
try:
|
||||
main.crawl_source(source_id)
|
||||
assert False, "Expected crawl failure"
|
||||
except RuntimeError:
|
||||
pass
|
||||
with db_module.connect() as db:
|
||||
assert db.execute("SELECT count(*) FROM events WHERE source_id=?", (source_id,)).fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_external_key_database_constraint(temp_db):
|
||||
import app.db as module
|
||||
with module.connect() as db:
|
||||
source_id = insert_source(db)
|
||||
values = (source_id, "same", "One", "2026-09-07T10:00:00+00:00", "Venue", "Address", "https://example.org/one")
|
||||
db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url) VALUES(?,?,?,?,?,?,?)", values)
|
||||
try:
|
||||
db.execute("INSERT INTO events(source_id,external_key,title,starts_at,venue_name,address,event_url) VALUES(?,?,?,?,?,?,?)", values)
|
||||
assert False, "Expected unique constraint"
|
||||
except sqlite3.IntegrityError:
|
||||
pass
|
||||
21
tests/unit/test_manifest.py
Normal file
21
tests/unit/test_manifest.py
Normal file
@ -0,0 +1,21 @@
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
def test_illawarra_manifest_contains_fifty_unique_https_sources(source_manifest):
|
||||
assert len(source_manifest) == 50
|
||||
urls = [item["url"] for item in source_manifest]
|
||||
assert len(set(urls)) == 50
|
||||
assert all(urlparse(url).scheme == "https" for url in urls)
|
||||
|
||||
|
||||
def test_updated_collegians_urls_are_present(source_manifest):
|
||||
urls = {item["url"] for item in source_manifest}
|
||||
assert "https://collegians.com.au/entertainment/" in urls
|
||||
assert "https://collegians.com.au/balgownie-entertainment/" in urls
|
||||
assert "https://www.collegians.com.au/whats-on/wollongong/" not in urls
|
||||
assert "https://www.collegians.com.au/whats-on/balgownie/" not in urls
|
||||
|
||||
|
||||
def test_manifest_has_expected_mode_diversity(source_manifest):
|
||||
modes = {item["mode"] for item in source_manifest}
|
||||
assert {"html", "dynamic", "recurring", "mixed", "image-heavy"} <= modes
|
||||
106
tests/unit/test_strategies.py
Normal file
106
tests/unit/test_strategies.py
Normal file
@ -0,0 +1,106 @@
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from app.strategies import (
|
||||
_events_from_html,
|
||||
_events_from_json,
|
||||
execute_strategy,
|
||||
json_path,
|
||||
parse_strategy,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_strategy_deep_copies_dict():
|
||||
original = {"mode": "html", "body": {"page": 1}}
|
||||
parsed = parse_strategy(original)
|
||||
parsed["body"]["page"] = 2
|
||||
assert original["body"]["page"] == 1
|
||||
assert parse_strategy(json.dumps(original)) == original
|
||||
assert parse_strategy("not json") == {}
|
||||
|
||||
|
||||
def test_json_path_handles_dicts_lists_and_defaults():
|
||||
value = {"results": [{"title": "One"}]}
|
||||
assert json_path(value, "results.0.title") == "One"
|
||||
assert json_path(value, "results.2.title", "missing") == "missing"
|
||||
assert json_path(value, "") is value
|
||||
|
||||
|
||||
def test_html_strategy_extracts_text_attributes_and_links(source, html_cards):
|
||||
strategy = {
|
||||
"item_selector": ".event",
|
||||
"title_selector": ".title",
|
||||
"date_selector": "time",
|
||||
"date_attribute": "datetime",
|
||||
"link_selector": ".title",
|
||||
"link_attribute": "href",
|
||||
"description_selector": ".description",
|
||||
"location_selector": ".venue",
|
||||
}
|
||||
events = _events_from_html(html_cards, strategy, source)
|
||||
assert [event.title for event in events] == ["Event One", "Event Two"]
|
||||
assert events[0].starts_at.hour == 19
|
||||
assert events[0].url == "https://example.org/events/one"
|
||||
assert events[0].description == "First description"
|
||||
assert events[0].address == "Town Hall"
|
||||
|
||||
|
||||
def test_html_scope_attribute_and_duration(source):
|
||||
html = '<article class="event" data-date="2026-09-07T19:00:00+10:00"><h2>One</h2></article>'
|
||||
strategy = {"item_selector": ".event", "title_selector": "h2", "date_selector": ":scope", "date_attribute": "data-date", "duration_minutes": 90}
|
||||
event = _events_from_html(html, strategy, source)[0]
|
||||
assert (event.ends_at - event.starts_at).total_seconds() == 5400
|
||||
|
||||
|
||||
def test_html_range_populates_end(source):
|
||||
html = '<article class="event"><h2>Festival</h2><time>2-13 Sep 2026</time></article>'
|
||||
event = _events_from_html(html, {"item_selector": ".event", "title_selector": "h2", "date_selector": "time"}, source)[0]
|
||||
assert event.starts_at.day == 2
|
||||
assert event.ends_at.day == 13
|
||||
|
||||
|
||||
def test_json_strategy_extracts_nested_fields(source):
|
||||
payload = {"results": [{"title": "Talk", "start": "2026-09-10T18:00:00+10:00", "end": "2026-09-10T20:00:00+10:00", "venue": {"address": "Library"}, "url": "/talk"}]}
|
||||
strategy = {"items_path": "results", "title_path": "title", "date_path": "start", "end_path": "end", "location_path": "venue.address", "link_path": "url"}
|
||||
event = _events_from_json(payload, strategy, source)[0]
|
||||
assert event.title == "Talk"
|
||||
assert event.ends_at.hour == 20
|
||||
assert event.address == "Library"
|
||||
|
||||
|
||||
def test_json_join_supplies_metadata_and_duration(source):
|
||||
payload = [{"movieId": "M1", "utcDate": "2026-09-11T10:00:00+00:00", "link": "/book"}]
|
||||
strategy = {"items_path": "", "date_path": "utcDate", "link_path": "link", "join_item_key": "movieId", "join_title_path": "name", "join_description_path": "summary", "join_duration_path": "duration"}
|
||||
event = _events_from_json(payload, strategy, source, {"M1": {"name": "Movie", "summary": "Plot", "duration": 120}})[0]
|
||||
assert event.title == "Movie"
|
||||
assert event.description == "Plot"
|
||||
assert (event.ends_at - event.starts_at).total_seconds() == 7200
|
||||
|
||||
|
||||
def test_disabled_json_item_is_skipped(source):
|
||||
payload = [{"title": "Hidden", "start": "2026-09-10", "disabled": True}]
|
||||
strategy = {"items_path": "", "title_path": "title", "date_path": "start", "disabled_path": "disabled"}
|
||||
assert _events_from_json(payload, strategy, source) == []
|
||||
|
||||
|
||||
def test_html_fragment_paginates(source, fake_response, recording_client, monkeypatch):
|
||||
import app.strategies as module
|
||||
monkeypatch.setattr(module, "validate_public_url", lambda url: None)
|
||||
responses = [
|
||||
fake_response(payload={"html": '<article class="event"><h2>One</h2><time>7 Sep 2026</time></article>', "pages": 2}),
|
||||
fake_response(payload={"html": '<article class="event"><h2>Two</h2><time>8 Sep 2026</time></article>', "pages": 2}),
|
||||
]
|
||||
client = recording_client(responses)
|
||||
source = {**source, "strategy_json": json.dumps({"mode": "html_fragment", "request_url": "https://example.org/ajax", "method": "POST", "body": {"action": "events"}, "html_path": "html", "item_selector": ".event", "title_selector": "h2", "date_selector": "time", "page_parameter": "page", "max_pages": 5, "total_pages_path": "pages"})}
|
||||
events = execute_strategy(source, client)
|
||||
assert [event.title for event in events] == ["One", "Two"]
|
||||
assert client.calls[0][2]["data"]["page"] == "1"
|
||||
assert client.calls[1][2]["data"]["page"] == "2"
|
||||
|
||||
|
||||
def test_unsupported_mode_raises(source, recording_client):
|
||||
source = {**source, "strategy_json": '{"mode":"magic"}'}
|
||||
with pytest.raises(Exception, match="Unsupported extraction mode"):
|
||||
execute_strategy(source, recording_client([]))
|
||||
Loading…
x
Reference in New Issue
Block a user