obligitory

.gitignore

@@ -4,6 +4,8 @@ __pycache__/
 .coverage
 htmlcov/
 test-results/*.txt
+tests/fixtures/source_cache/*
+!tests/fixtures/source_cache/.gitkeep
 .venv/
 data/
 *.db

Dockerfile

@@ -5,6 +5,7 @@ COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 COPY app ./app
 COPY tests ./tests
+COPY scripts ./scripts
 COPY pytest.ini .
 COPY README.md STRATEGIES.md ./
 RUN mkdir -p /data

README.md

@@ -45,6 +45,28 @@ docker compose --profile test run --rm test
 
 Detailed per-test outcomes, durations, failure traces, and totals are exported to `test-results/latest.txt` after every run. The file is replaced on the next run and is ignored by Git.
 
+### Cached source extraction corpus
+
+Caching is manual so normal tests never contact source websites. The cache command requests each manifest URL once, waits two seconds between requests, skips already cached documents, and records fetch metadata in `tests/fixtures/source_cache/index.json`:
+
+```sh
+docker compose --profile test run --rm cache-sources
+```
+
+Use `--force` only when intentionally refreshing every document. You can trial a smaller batch with `--limit`, for example:
+
+```sh
+docker compose --profile test run --rm cache-sources python scripts/cache_source_documents.py --delay 2 --limit 5
+```
+
+After caching, run automatic extraction entirely offline:
+
+```sh
+docker compose --profile test run --rm test sh -c "pip install --no-cache-dir pytest==8.4.2 pytest-cov==6.2.1 && python -m pytest tests/unit/test_cached_source_extraction.py -m cached_sources -vv -o addopts='' --extraction-results=/test-results/source-extraction.txt"
+```
+
+Per-source event counts and extraction failures are written to `test-results/source-extraction.txt`. Cached website documents and generated reports remain local and are ignored by Git.
+
 Run optional live reachability checks separately (results depend on third-party availability and access policies):
 
 ```sh

compose.yaml

@@ -18,10 +18,21 @@ services:
     command: ["sh", "-c", "pip install --no-cache-dir pytest==8.4.2 pytest-cov==6.2.1 && python -m pytest tests/unit -vv -rA --tb=short --cov=app --cov-report=term-missing --results-file=/test-results/latest.txt"]
     volumes:
       - ./test-results:/test-results
+      - ./tests/fixtures/source_cache:/app/tests/fixtures/source_cache
     environment:
       DATABASE_PATH: /tmp/what2do-test.db
       PYTHONPATH: /app
     profiles: ["test"]
 
+  cache-sources:
+    build: .
+    working_dir: /app
+    command: ["python", "scripts/cache_source_documents.py", "--delay", "2"]
+    volumes:
+      - ./tests/fixtures/source_cache:/app/tests/fixtures/source_cache
+    environment:
+      PYTHONPATH: /app
+    profiles: ["test"]
+
 volumes:
   what2do-data:

pytest.ini

@@ -1,6 +1,7 @@
 [pytest]
 testpaths = tests
-addopts = -ra -m "not network"
+addopts = -ra -m "not network and not cached_sources"
 pythonpath = .
 markers =
     network: live-network integration tests, excluded from normal runs
+    cached_sources: offline extraction tests using manually cached source documents

scripts/cache_source_documents.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+import argparse
+import hashlib
+import json
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+from app.crawler import _http_client
+
+
+ROOT = Path(__file__).resolve().parents[1]
+MANIFEST_PATH = ROOT / "tests" / "fixtures" / "illawarra_sources.json"
+CACHE_DIR = ROOT / "tests" / "fixtures" / "source_cache"
+INDEX_PATH = CACHE_DIR / "index.json"
+
+
+def cache_key(source):
+    digest = hashlib.sha256(source["url"].encode()).hexdigest()[:12]
+    return f"{digest}.html"
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Manually cache each Illawarra source once for offline extraction tests.")
+    parser.add_argument("--delay", type=float, default=2.0, help="Seconds to wait between source requests (default: 2).")
+    parser.add_argument("--timeout", type=float, default=30.0, help="Per-request timeout in seconds (default: 30).")
+    parser.add_argument("--limit", type=int, help="Only fetch the first N sources, useful for a trial run.")
+    parser.add_argument("--force", action="store_true", help="Replace documents already present in the cache.")
+    args = parser.parse_args()
+
+    sources = json.loads(MANIFEST_PATH.read_text())
+    if args.limit is not None:
+        sources = sources[:args.limit]
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    previous = json.loads(INDEX_PATH.read_text()) if INDEX_PATH.exists() else {"sources": {}}
+    records = previous.get("sources", {})
+
+    with _http_client(timeout=args.timeout) as client:
+        for position, source in enumerate(sources, start=1):
+            filename = cache_key(source)
+            destination = CACHE_DIR / filename
+            existing = records.get(source["url"])
+            if destination.exists() and existing and not args.force:
+                print(f"[{position}/{len(sources)}] SKIP {source['name']} (already cached)")
+                continue
+            print(f"[{position}/{len(sources)}] GET  {source['name']} — {source['url']}")
+            record = {"name": source["name"], "url": source["url"], "mode": source["mode"], "filename": filename, "fetched_at": datetime.now(timezone.utc).isoformat()}
+            try:
+                response = client.get(source["url"])
+                record.update({"status_code": response.status_code, "content_type": response.headers.get("content-type", ""), "final_url": str(response.url), "bytes": len(response.content)})
+                if response.status_code < 400 and "html" in record["content_type"].lower():
+                    destination.write_text(response.text, encoding="utf-8")
+                    record["cached"] = True
+                    print(f"             SAVED {filename} ({record['bytes']} bytes)")
+                else:
+                    record["cached"] = False
+                    record["error"] = f"Expected successful HTML response, got {response.status_code} {record['content_type'] or 'unknown content type'}"
+                    destination.unlink(missing_ok=True)
+                    print(f"             ERROR {record['error']}")
+            except Exception as exc:
+                record.update({"cached": False, "error": f"{type(exc).__name__}: {exc}"})
+                destination.unlink(missing_ok=True)
+                print(f"             ERROR {record['error']}")
+            records[source["url"]] = record
+            INDEX_PATH.write_text(json.dumps({"generated_at": datetime.now(timezone.utc).isoformat(), "sources": records}, indent=2) + "\n")
+            if position < len(sources) and args.delay > 0:
+                time.sleep(args.delay)
+
+    cached = sum(bool(record.get("cached")) for record in records.values())
+    print(f"Cache complete: {cached}/{len(records)} documents available in {CACHE_DIR}")
+
+
+if __name__ == "__main__":
+    main()

tests/conftest.py

@@ -4,8 +4,12 @@ from pathlib import Path
 import pytest
 
 
+EXTRACTION_RESULTS = []
+
+
 def pytest_addoption(parser):
     parser.addoption("--results-file", help="Write the human-readable terminal report to this file.")
+    parser.addoption("--extraction-results", help="Write cached source extraction counts to this file.")
 
 
 def pytest_configure(config):
@@ -14,6 +18,22 @@ def pytest_configure(config):
         config.pluginmanager.register(_ResultsFilePlugin(Path(results_file)), "results-file")
 
 
+def pytest_sessionfinish(session, exitstatus):
+    report_option = session.config.getoption("--extraction-results")
+    if not report_option or not EXTRACTION_RESULTS:
+        return
+    report_path = Path(report_option)
+    rows = sorted(EXTRACTION_RESULTS, key=lambda row: row["name"].casefold())
+    lines = ["What2Do cached source extraction results", "=" * 39, "", f"{'Source':40} {'Mode':12} {'Events':>6}  Status", "-" * 75]
+    for row in rows:
+        lines.append(f"{row['name'][:40]:40} {row['mode'][:12]:12} {row['events']:6}  {row['status'].upper()}")
+        if row["message"]:
+            lines.append(f"  {row['message']}")
+    lines.extend(("", f"Sources tested: {len(rows)}", f"Events found: {sum(row['events'] for row in rows)}", f"Successful extractions: {sum(row['status'] == 'passed' for row in rows)}", f"Failed extractions: {sum(row['status'] == 'failed' for row in rows)}"))
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    report_path.write_text("\n".join(lines) + "\n")
+
+
 class _ResultsFilePlugin:
     def __init__(self, path):
         self.path = path
@@ -36,6 +56,11 @@ class _ResultsFilePlugin:
         self.path.write_text("\n".join(lines) + "\n")
 
 
+@pytest.fixture
+def record_extraction_result():
+    return EXTRACTION_RESULTS.append
+
+
 @pytest.fixture
 def source():
     return {

tests/unit/test_cached_source_extraction.py

@@ -0,0 +1,41 @@
+import json
+from pathlib import Path
+
+import pytest
+
+from app.crawler import CrawlError, detect_rules
+
+
+CACHE_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "source_cache"
+INDEX_PATH = CACHE_DIR / "index.json"
+
+
+
+def cached_sources():
+    if not INDEX_PATH.exists():
+        return []
+    index = json.loads(INDEX_PATH.read_text())
+    return [record for record in index.get("sources", {}).values() if record.get("cached")]
+
+
+@pytest.mark.cached_sources
+@pytest.mark.parametrize("record", cached_sources(), ids=lambda record: record["name"])
+def test_automatic_extraction_from_cached_source(record, record_extraction_result):
+    document = CACHE_DIR / record["filename"]
+    result = {"name": record["name"], "url": record["url"], "mode": record["mode"], "document": record["filename"], "events": 0, "status": "failed", "message": ""}
+    try:
+        rules, events, message = detect_rules(document.read_text(encoding="utf-8", errors="replace"), record["url"])
+        result.update({"events": len(events), "status": "passed" if events else "failed", "message": message, "item_selector": rules.get("item_selector", "")})
+        assert events, "Automatic extraction returned no events"
+    except CrawlError as exc:
+        result["message"] = str(exc)
+        pytest.fail(str(exc))
+    finally:
+        record_extraction_result(result)
+
+
+@pytest.mark.cached_sources
+def test_source_cache_exists():
+    assert INDEX_PATH.exists(), "No source cache exists. Run: python scripts/cache_source_documents.py"
+    assert cached_sources(), "The source cache index contains no successful HTML documents."
+