64 lines
3.1 KiB
Python
64 lines
3.1 KiB
Python
import json
|
|
|
|
import pytest
|
|
|
|
from app.crawler import detect_rules, extract_events
|
|
|
|
|
|
def base_source(**overrides):
|
|
value = {"url": "https://example.org/events", "timezone": "Australia/Sydney", "item_selector": "json-ld", "title_selector": "", "date_selector": "", "time_selector": "", "link_selector": "", "description_selector": "", "location_selector": "", "date_format": "", "recurring_weekly": 0}
|
|
value.update(overrides)
|
|
return value
|
|
|
|
|
|
def test_jsonld_extracts_graph_event_with_end_address_and_relative_url():
|
|
payload = {"@graph": [{"@type": "Event", "name": "Gallery Talk", "startDate": "2026-09-07T18:00:00+10:00", "endDate": "2026-09-07T20:00:00+10:00", "description": "A talk", "url": "/talk", "location": {"address": {"streetAddress": "1 Art St", "addressLocality": "Wollongong", "addressRegion": "NSW", "postalCode": "2500"}}}]}
|
|
html = f'<script type="application/ld+json">{json.dumps(payload)}</script>'
|
|
event = extract_events(html, base_source())[0]
|
|
assert event.title == "Gallery Talk"
|
|
assert event.ends_at.hour == 20
|
|
assert event.address == "1 Art St, Wollongong, NSW, 2500"
|
|
assert event.url == "https://example.org/talk"
|
|
|
|
|
|
def test_jsonld_ignores_invalid_and_missing_dates():
|
|
html = '''<script type="application/ld+json">not json</script>
|
|
<script type="application/ld+json">[{"@type":"Event","name":"No date"},{"@type":"Event","name":"Good","startDate":"2026-09-08"}]</script>'''
|
|
events = extract_events(html, base_source())
|
|
assert [event.title for event in events] == ["Good"]
|
|
|
|
|
|
def test_html_extraction_skips_invalid_sibling():
|
|
html = '''<article><h2>Bad</h2><time>not a date</time></article>
|
|
<article><h2>Good</h2><time>9 Sep 2026 7pm</time><a href="/good">Details</a></article>'''
|
|
source = base_source(item_selector="article", title_selector="h2", date_selector="time", link_selector="a")
|
|
events = extract_events(html, source)
|
|
assert len(events) == 1
|
|
assert events[0].title == "Good"
|
|
|
|
|
|
def test_detector_prefers_jsonld():
|
|
html = '<script type="application/ld+json">{"@type":"Event","name":"One","startDate":"2026-09-08"}</script>'
|
|
rules, events, message = detect_rules(html, "https://example.org/events")
|
|
assert rules["item_selector"] == "json-ld"
|
|
assert len(events) == 1
|
|
assert "JSON-LD" in message
|
|
|
|
|
|
def test_detector_finds_repeated_html_cards():
|
|
html = '''<article class="event"><h2 class="title">One</h2><time>8 Sep 2027 7pm</time><a href="/one">More</a></article>
|
|
<article class="event"><h2 class="title">Two</h2><time>9 Sep 2027 8pm</time><a href="/two">More</a></article>'''
|
|
rules, events, message = detect_rules(html, "https://example.org/events")
|
|
assert len(events) >= 2
|
|
assert rules["item_selector"]
|
|
assert rules["title_selector"]
|
|
assert rules["date_selector"]
|
|
assert "Guessed CSS" in message
|
|
|
|
|
|
def test_detector_rejects_social_feed_false_positive():
|
|
html = '''<div class="sbi_item"><a>Party this Friday</a><span>Friday night</span></div>
|
|
<div class="sbi_item"><a>More next Friday</a><span>Friday night</span></div>'''
|
|
with pytest.raises(Exception, match="No repeated event-like"):
|
|
detect_rules(html, "https://example.org/events")
|