diff --git a/.gitignore b/.gitignore index d5624cb..f38b3f6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ __pycache__/ .coverage htmlcov/ test-results/*.txt +tests/fixtures/source_cache/* +!tests/fixtures/source_cache/.gitkeep .venv/ data/ *.db diff --git a/Dockerfile b/Dockerfile index 31ab55b..8c72f65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY app ./app COPY tests ./tests +COPY scripts ./scripts COPY pytest.ini . COPY README.md STRATEGIES.md ./ RUN mkdir -p /data diff --git a/README.md b/README.md index 7e2dadd..352f8c5 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,28 @@ docker compose --profile test run --rm test Detailed per-test outcomes, durations, failure traces, and totals are exported to `test-results/latest.txt` after every run. The file is replaced on the next run and is ignored by Git. +### Cached source extraction corpus + +Caching is manual so normal tests never contact source websites. The cache command requests each manifest URL once, waits two seconds between requests, skips already cached documents, and records fetch metadata in `tests/fixtures/source_cache/index.json`: + +```sh +docker compose --profile test run --rm cache-sources +``` + +Use `--force` only when intentionally refreshing every document. You can trial a smaller batch with `--limit`, for example: + +```sh +docker compose --profile test run --rm cache-sources python scripts/cache_source_documents.py --delay 2 --limit 5 +``` + +After caching, run automatic extraction entirely offline: + +```sh +docker compose --profile test run --rm test sh -c "pip install --no-cache-dir pytest==8.4.2 pytest-cov==6.2.1 && python -m pytest tests/unit/test_cached_source_extraction.py -m cached_sources -vv -o addopts='' --extraction-results=/test-results/source-extraction.txt" +``` + +Per-source event counts and extraction failures are written to `test-results/source-extraction.txt`. Cached website documents and generated reports remain local and are ignored by Git. + Run optional live reachability checks separately (results depend on third-party availability and access policies): ```sh diff --git a/app/__pycache__/crawler.cpython-314.pyc b/app/__pycache__/crawler.cpython-314.pyc index 102872f..f77d763 100644 Binary files a/app/__pycache__/crawler.cpython-314.pyc and b/app/__pycache__/crawler.cpython-314.pyc differ diff --git a/app/__pycache__/main.cpython-314.pyc b/app/__pycache__/main.cpython-314.pyc index 9062bd6..c65c3a5 100644 Binary files a/app/__pycache__/main.cpython-314.pyc and b/app/__pycache__/main.cpython-314.pyc differ diff --git a/compose.yaml b/compose.yaml index 1523060..66c7571 100644 --- a/compose.yaml +++ b/compose.yaml @@ -18,10 +18,21 @@ services: command: ["sh", "-c", "pip install --no-cache-dir pytest==8.4.2 pytest-cov==6.2.1 && python -m pytest tests/unit -vv -rA --tb=short --cov=app --cov-report=term-missing --results-file=/test-results/latest.txt"] volumes: - ./test-results:/test-results + - ./tests/fixtures/source_cache:/app/tests/fixtures/source_cache environment: DATABASE_PATH: /tmp/what2do-test.db PYTHONPATH: /app profiles: ["test"] + cache-sources: + build: . + working_dir: /app + command: ["python", "scripts/cache_source_documents.py", "--delay", "2"] + volumes: + - ./tests/fixtures/source_cache:/app/tests/fixtures/source_cache + environment: + PYTHONPATH: /app + profiles: ["test"] + volumes: what2do-data: diff --git a/pytest.ini b/pytest.ini index 3a2bd06..fdc7405 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,7 @@ [pytest] testpaths = tests -addopts = -ra -m "not network" +addopts = -ra -m "not network and not cached_sources" pythonpath = . markers = network: live-network integration tests, excluded from normal runs + cached_sources: offline extraction tests using manually cached source documents diff --git a/scripts/cache_source_documents.py b/scripts/cache_source_documents.py new file mode 100644 index 0000000..61c7175 --- /dev/null +++ b/scripts/cache_source_documents.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import time +from datetime import datetime, timezone +from pathlib import Path + +from app.crawler import _http_client + + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST_PATH = ROOT / "tests" / "fixtures" / "illawarra_sources.json" +CACHE_DIR = ROOT / "tests" / "fixtures" / "source_cache" +INDEX_PATH = CACHE_DIR / "index.json" + + +def cache_key(source): + digest = hashlib.sha256(source["url"].encode()).hexdigest()[:12] + return f"{digest}.html" + + +def main(): + parser = argparse.ArgumentParser(description="Manually cache each Illawarra source once for offline extraction tests.") + parser.add_argument("--delay", type=float, default=2.0, help="Seconds to wait between source requests (default: 2).") + parser.add_argument("--timeout", type=float, default=30.0, help="Per-request timeout in seconds (default: 30).") + parser.add_argument("--limit", type=int, help="Only fetch the first N sources, useful for a trial run.") + parser.add_argument("--force", action="store_true", help="Replace documents already present in the cache.") + args = parser.parse_args() + + sources = json.loads(MANIFEST_PATH.read_text()) + if args.limit is not None: + sources = sources[:args.limit] + CACHE_DIR.mkdir(parents=True, exist_ok=True) + previous = json.loads(INDEX_PATH.read_text()) if INDEX_PATH.exists() else {"sources": {}} + records = previous.get("sources", {}) + + with _http_client(timeout=args.timeout) as client: + for position, source in enumerate(sources, start=1): + filename = cache_key(source) + destination = CACHE_DIR / filename + existing = records.get(source["url"]) + if destination.exists() and existing and not args.force: + print(f"[{position}/{len(sources)}] SKIP {source['name']} (already cached)") + continue + print(f"[{position}/{len(sources)}] GET {source['name']} — {source['url']}") + record = {"name": source["name"], "url": source["url"], "mode": source["mode"], "filename": filename, "fetched_at": datetime.now(timezone.utc).isoformat()} + try: + response = client.get(source["url"]) + record.update({"status_code": response.status_code, "content_type": response.headers.get("content-type", ""), "final_url": str(response.url), "bytes": len(response.content)}) + if response.status_code < 400 and "html" in record["content_type"].lower(): + destination.write_text(response.text, encoding="utf-8") + record["cached"] = True + print(f" SAVED {filename} ({record['bytes']} bytes)") + else: + record["cached"] = False + record["error"] = f"Expected successful HTML response, got {response.status_code} {record['content_type'] or 'unknown content type'}" + destination.unlink(missing_ok=True) + print(f" ERROR {record['error']}") + except Exception as exc: + record.update({"cached": False, "error": f"{type(exc).__name__}: {exc}"}) + destination.unlink(missing_ok=True) + print(f" ERROR {record['error']}") + records[source["url"]] = record + INDEX_PATH.write_text(json.dumps({"generated_at": datetime.now(timezone.utc).isoformat(), "sources": records}, indent=2) + "\n") + if position < len(sources) and args.delay > 0: + time.sleep(args.delay) + + cached = sum(bool(record.get("cached")) for record in records.values()) + print(f"Cache complete: {cached}/{len(records)} documents available in {CACHE_DIR}") + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index d192ce9..b1315c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,8 +4,12 @@ from pathlib import Path import pytest +EXTRACTION_RESULTS = [] + + def pytest_addoption(parser): parser.addoption("--results-file", help="Write the human-readable terminal report to this file.") + parser.addoption("--extraction-results", help="Write cached source extraction counts to this file.") def pytest_configure(config): @@ -14,6 +18,22 @@ def pytest_configure(config): config.pluginmanager.register(_ResultsFilePlugin(Path(results_file)), "results-file") +def pytest_sessionfinish(session, exitstatus): + report_option = session.config.getoption("--extraction-results") + if not report_option or not EXTRACTION_RESULTS: + return + report_path = Path(report_option) + rows = sorted(EXTRACTION_RESULTS, key=lambda row: row["name"].casefold()) + lines = ["What2Do cached source extraction results", "=" * 39, "", f"{'Source':40} {'Mode':12} {'Events':>6} Status", "-" * 75] + for row in rows: + lines.append(f"{row['name'][:40]:40} {row['mode'][:12]:12} {row['events']:6} {row['status'].upper()}") + if row["message"]: + lines.append(f" {row['message']}") + lines.extend(("", f"Sources tested: {len(rows)}", f"Events found: {sum(row['events'] for row in rows)}", f"Successful extractions: {sum(row['status'] == 'passed' for row in rows)}", f"Failed extractions: {sum(row['status'] == 'failed' for row in rows)}")) + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text("\n".join(lines) + "\n") + + class _ResultsFilePlugin: def __init__(self, path): self.path = path @@ -36,6 +56,11 @@ class _ResultsFilePlugin: self.path.write_text("\n".join(lines) + "\n") +@pytest.fixture +def record_extraction_result(): + return EXTRACTION_RESULTS.append + + @pytest.fixture def source(): return { diff --git a/tests/fixtures/source_cache/.gitkeep b/tests/fixtures/source_cache/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_cached_source_extraction.py b/tests/unit/test_cached_source_extraction.py new file mode 100644 index 0000000..1c878cd --- /dev/null +++ b/tests/unit/test_cached_source_extraction.py @@ -0,0 +1,41 @@ +import json +from pathlib import Path + +import pytest + +from app.crawler import CrawlError, detect_rules + + +CACHE_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "source_cache" +INDEX_PATH = CACHE_DIR / "index.json" + + + +def cached_sources(): + if not INDEX_PATH.exists(): + return [] + index = json.loads(INDEX_PATH.read_text()) + return [record for record in index.get("sources", {}).values() if record.get("cached")] + + +@pytest.mark.cached_sources +@pytest.mark.parametrize("record", cached_sources(), ids=lambda record: record["name"]) +def test_automatic_extraction_from_cached_source(record, record_extraction_result): + document = CACHE_DIR / record["filename"] + result = {"name": record["name"], "url": record["url"], "mode": record["mode"], "document": record["filename"], "events": 0, "status": "failed", "message": ""} + try: + rules, events, message = detect_rules(document.read_text(encoding="utf-8", errors="replace"), record["url"]) + result.update({"events": len(events), "status": "passed" if events else "failed", "message": message, "item_selector": rules.get("item_selector", "")}) + assert events, "Automatic extraction returned no events" + except CrawlError as exc: + result["message"] = str(exc) + pytest.fail(str(exc)) + finally: + record_extraction_result(result) + + +@pytest.mark.cached_sources +def test_source_cache_exists(): + assert INDEX_PATH.exists(), "No source cache exists. Run: python scripts/cache_source_documents.py" + assert cached_sources(), "The source cache index contains no successful HTML documents." +