From 65c1e3ceac3f67c6813222bd91a20627b9b6e619 Mon Sep 17 00:00:00 2001
From: "zahkcb@gmail.com" <zahkcb@gmail.com>
Date: Mon, 7 Sep 2026 12:53:26 +1000
Subject: [PATCH] obligitory

---
 .gitignore                                  |   2 +
 Dockerfile                                  |   1 +
 README.md                                   |  22 ++++++
 app/__pycache__/crawler.cpython-314.pyc     | Bin 30012 -> 30227 bytes
 app/__pycache__/main.cpython-314.pyc        | Bin 62298 -> 66108 bytes
 compose.yaml                                |  11 +++
 pytest.ini                                  |   3 +-
 scripts/cache_source_documents.py           |  74 ++++++++++++++++++++
 tests/conftest.py                           |  25 +++++++
 tests/fixtures/source_cache/.gitkeep        |   0
 tests/unit/test_cached_source_extraction.py |  41 +++++++++++
 11 files changed, 178 insertions(+), 1 deletion(-)
 create mode 100644 scripts/cache_source_documents.py
 create mode 100644 tests/fixtures/source_cache/.gitkeep
 create mode 100644 tests/unit/test_cached_source_extraction.py

diff --git a/.gitignore b/.gitignore
index d5624cb..f38b3f6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,8 @@ __pycache__/
 .coverage
 htmlcov/
 test-results/*.txt
+tests/fixtures/source_cache/*
+!tests/fixtures/source_cache/.gitkeep
 .venv/
 data/
 *.db
diff --git a/Dockerfile b/Dockerfile
index 31ab55b..8c72f65 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,6 +5,7 @@ COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 COPY app ./app
 COPY tests ./tests
+COPY scripts ./scripts
 COPY pytest.ini .
 COPY README.md STRATEGIES.md ./
 RUN mkdir -p /data
diff --git a/README.md b/README.md
index 7e2dadd..352f8c5 100644
--- a/README.md
+++ b/README.md
@@ -45,6 +45,28 @@ docker compose --profile test run --rm test
 
 Detailed per-test outcomes, durations, failure traces, and totals are exported to `test-results/latest.txt` after every run. The file is replaced on the next run and is ignored by Git.
 
+### Cached source extraction corpus
+
+Caching is manual so normal tests never contact source websites. The cache command requests each manifest URL once, waits two seconds between requests, skips already cached documents, and records fetch metadata in `tests/fixtures/source_cache/index.json`:
+
+```sh
+docker compose --profile test run --rm cache-sources
+```
+
+Use `--force` only when intentionally refreshing every document. You can trial a smaller batch with `--limit`, for example:
+
+```sh
+docker compose --profile test run --rm cache-sources python scripts/cache_source_documents.py --delay 2 --limit 5
+```
+
+After caching, run automatic extraction entirely offline:
+
+```sh
+docker compose --profile test run --rm test sh -c "pip install --no-cache-dir pytest==8.4.2 pytest-cov==6.2.1 && python -m pytest tests/unit/test_cached_source_extraction.py -m cached_sources -vv -o addopts='' --extraction-results=/test-results/source-extraction.txt"
+```
+
+Per-source event counts and extraction failures are written to `test-results/source-extraction.txt`. Cached website documents and generated reports remain local and are ignored by Git.
+
 Run optional live reachability checks separately (results depend on third-party availability and access policies):
 
 ```sh
diff --git a/app/__pycache__/crawler.cpython-314.pyc b/app/__pycache__/crawler.cpython-314.pyc
index 102872f1d57de802b20441677f94e117d4cb97f6..f77d76339926fb603b683f1dc09d31f08b7354b9 100644
GIT binary patch
delta 2294
zcmZXWdr(tX8o<wYlUyKy@Ca!l0!d5=MgvKy1%#qqDi~C>Mh#>al`Mppkfi4Z1sr$n
zOr4?AZtKjc(`xOGqjk5Pb~~)s-Hi_0?sRK+$L@9jJA=K`DV_G8zPAoVw(p+rMyze`
zOn&z}=X;)W?ss7FLtr0+akVi=Px#a3`29fn-G4Sdr(JocuP;6n^^HaP_I3F-#Jh%v
zqOqjpOVId0w2KP|`Xb4y?zm6BsP!#dVFAxI!tt2&8NQz0puV0TQD2h*u49JM65$;U
zd)wk?pWT<o21<|Zk_jm$A#EijbRk_HX?~WFgc#I?#I?MZdML+|_|q$*os9&=v?h@e
z%2?8MYf!h7wC06KHVKlDz@|Ng?rc(}No4-aOl;MLL~fQ*B_y&svY8{RSaguK2iFK}
z`Mx51NE6JgfppS@{s-^gdQ8%Gn*G&Q0ea@xS16?OFrz&;M}Lcpo~^R41_KM(osiAO
z>=ls19D|m7j6Vbp<_3+yJpRiM8hx5zL5=oiLjAp^Y57+6nABZT)8_r8LUpn1?R#^1
zReafOk0YP`d%(o%wwu^7M`4a>&5hOcK`0$y4+$17)$1f4GgBOH_n1%?=D*P~jlC@F
z0YyxX^~ZYR&w`@uj*Lnkp!f5Nm}AAi%qUQ-Sjm)<G?DBdiqie;n-%qZ(QBQ)ZAxZe
zMA|zVjZnpqh)^lIcYkDXII3i)2R(5*6iL!L9zZg1xZQ61jfa+C=z~k>Mw8R*QD;Zw
zs%7~Tqtlj^vzCS_OT#l8CR(Pw4da11=j!8S6HOB}DM#v_mjV|8sk*7#o5wfLt@IyX
zn_8AK{Ns+P>W$-D-ZGcWn%y&I_o;(d%)gj5w@jN`rnc{xGPhhY?;LNQGc9LcSEZOw
zid>%;$epey0S?Juml;L)12cHnzzghduM>VNcY6&29F-H5%>d7_zxg)7Stj`X<_RqM
z0mKo+GQ<mPqrXD;8W#`AqyCQs9%{PQ1}EercbpWQ=g=g@p=kQMN9cdKB;lU&y*QlS
zhVyJgbs3yw1J#|GuXBwjgHExT>H{XUr3nO1E6`6<tfQt;_dXw;kk8b#0-TYr)&2)8
zYLY=dEeRZM`%>UH>dN4c?8~}NNU>kl*XicD_KbX@z7u$ntr3@>cOj8lI1ZCV5&#&0
zYv~s@w>o~>bVE3=;R4&%Pyzuq)L?^)^6wfR(Sl1ZZd}kTcky<<Eo2HhyhS|PjVF&Z
zL307~HkX%w!S&XV^=2#cD~MkQ){v#Pose{6Z!RQ6Zf|}KY+n(QMM}BzX7f#nd#ZDq
z&E;}qKn9qUueW?66wmNnv;y%zj-RamyDgP)fmODyhELcpTT9>^8)+>S&vQe?#_r!?
z$?A>9q9X~q?t(nodI@qd81I0s_|njG?C7>qJz7&8@2wd*we5ANnZ@zr2<#6uL-{fS
z&r0H#6hDEbv&Ct2eoy}U_ICk3V5fFi;ZO2~9V@}EI(DMZk4|mL<x?4UEkcjTy*;9$
z#r9LQQX&#lG~NCCsfEweEDl8&9^_^8qF*1PpTnc2#b{81C|xr32ozI&z{gnmElReh
ztyVOn-HYsG+a}Yi7-$uTB1&<ZjCRv`_C=f5GKZdSx&_r%#6yU)?5_3^SY&@`Z(7Aq
zn$Ry1{1}OnMWbEAl=jDZ6&)QOj7o|a-H(4>=w;>#AL3WZ>2RaIiDy>7#fRlj!+vef
zoW=D_Nvh%v&&D^ccDZ!VI=C%^xYvKKAuI0WuvK025mx{f1B~vSg#&C;r#-j@NX1r=
z0zB)>Ir;6?!prM(!&%}#bFIi#HqKX!BGT&#8^udSBGP?(*d)GMEOO4&n#0zDtJQ|E
zsC!cfVUbOC9!)n2%*!<L8*SrtT$j4!hf!KY<Rku%P&eeq>!;tK;#<Vmi0@cckBfhk
zZ9T5tcxBKR5x+(ZA+Wcpy7_r1jU!GVE^&DB(;FYhiL;2O5b717mQsla*noJRUF`|#
z9VjB^<u$zr1<02t`a3jmo#hN|1S{J$bdg^Sd9ke(ZZyomU{#3Y2>copT`!FfCp?B*
zPW&u;C>GQ0<l21s+t?X^%j|T*4-WZCq8&iPymURE8l+C$Zf@gXkI`2QVT@mcQD0(R
z(h)-$j@1=9*}UYnsav709j_>gpMzRA#WZv`TbC?>0oIY6@bmvT7OIM%V6iR()-4oh
dp$QhV)thQzr%((93wIe|FD%*xsHS!v>wBIgMri;5

delta 2058
zcmZ9Nc}!GS6o=oLH}hs>oM9OrvJNxMFgmhoK?ISliXfHXw1t7VfPjd~;vKLqwJxpI
zq>Z(?O$*wl6>aL)sC~6I)jy1TrB$m~&BUZj+SFQGF-pg7NzZveKwmQ7J?GqWfA`#X
z?|sAX!ihUzPBNQ<1blXWvSVps{t@#5Etr)bv}sWVGPv&xh_`(G)a;p|HV0R!&507V
zkJ)f&y+{H>j}cq)9~|Tr)^Fu*L5LM9wW-jqtJH-E#cv2gyTspQwxB90Lo+yrU#XXq
zM2{{D2K{!uN2-*v(Q8;h!#(;TuSyluQ@*IF2;sY|7L#FQR2Ac)wiFoYud>-k_gCuq
zW?K>j@gp_|82NR~9m_A(M@;aTMke=|(F?|3h{u#BdP1|b!$slnLVL`heCT84R#Gu<
z&$WwsY%k4?855-zO3_|>wka%QXkkkLHLlc$hb$00mf^Y`f;*gNCb;Zw@Jk(Qnp+pQ
zt<fY_<8QSa*hZXvANaKmb*p`RW5Riv%JCa~9jv{hxuub<M3-&k^BvQ`z`Gr3CYFN{
zp8?@=x%hvMDA>fuIp-DLlH+d3uB)=^gU)O6)EjcaH*&!R%@w)enq0QE_-3%BE4we)
zxi|c9b)P%$W>oz9)^6+GXAgPzdAqFFqNZ$@c%L&>G6~Vnhi1i)xD#Nlvd3kXU^iEi
zGvENfo$P>h%Gea62DT}YX~h70`NH(s(8Kqqr^_Exov#o!6YvEI>;S))o}#;e=32!*
z={F5Z73$-mOG(LiSK~NDDMt~0vAJOtyNH&L7VomzHn2GAJi_lzNrW9dEUVVgivhQu
zea+Woy<nkS)=r?XADY=wuAN$>`x(71rEKaPfL%&=_E#`Yon$$NeRLLGwqfBHrzOIN
zyk}Z19OYtej;<ejcPT}=wIJ<AlZ8;F=^xR+Qp5%!AmD<R!u`R9u8ivMu?!iXifHh_
z^f)Nyavm)*GjE#~EXtXp0Wrpcw<4m_;;|=3quS6DBFuzh;C;p8b+@rUQc0NA3-NaZ
zVT@qKt}=WXLsrC)C6+}fV@nl)1Io$SziVPIqY_IY+(kUG#4qQh!ZH46PEsC93gvE*
zby8+I!f&Xr^EEDRTi$?CdPRPzrH(CYWULk)A6;+P$R^a4JK>n}YWZ0RrU~)Tk_L_B
zH5JyNBj_*<-heC0rix3D)kpq21UlP86R`6Xpe=hsox(KuJ7tUKM~J2!R1fzC;bh~5
z!gEX14-ihdUwQet31CzAvzC$`+s|Q@J4)~K0kDcF=$8Cix<P)euf49-FE%u<WZ@WR
z692v`CMAl@XhMt{M7TAKM1Mt-${y9pCwQkyVU%~AFZRy1oTm9EBK(rCjde6Ouzr5p
zn+&&jzgNZ!Ayuz}hkSkYOyi%F86Z63m#eLCp8rw34sY`Enxdd06jV>xdgb$)bgeS5
zAP*u$!Il3|6cREK@ssbT3C<D-%;2T9H^9g3bv9242r1>@D?!aurciI9=2U(#R(Co$
zk~lFQdB!Xelil9w(%EQ<=$cfmk<P_R$gAV!YJJ$vM58xTcQXULnVi>cA9d>Co>IE#
zC2i3Is`@p;>x74dP{IHqk}zs@<o-?ghwvBSUp}VE37_%QCg*(m2(aUXMT8at?KM+h
zWe8bY2|Ea<5bjX+4B2gj9>RMB_0p@QB++){5ccsyO`aetnS^_aamiW@n3OF`=ZWxu
z|I{)IWS-pm1zz=ETFX-$l&B}M$priZ6BzBSU)RLiI@{gG!HLxc_V7h*t-86`Yf`Sa
z?FQVx)hp5=PT9Z03m|aOmye;ReGXj>=EU)3zB9&3($6B*6FjG5voV2ub%i#5u_Jl9
z`YP15_fry`0<~^N(=gIucIvPZ=sV~)<hHambS`hqV@>?2&co^WXFZS_4Pk*CJ>(69
dX<;S=#;TvAfeKA5gbn1HVIc%;8knN0{0~rV|2hBw

diff --git a/app/__pycache__/main.cpython-314.pyc b/app/__pycache__/main.cpython-314.pyc
index 9062bd69c3360c7ce1dad50b66dbe95ac28eadd1..c65c3a570c086ed02f2bad563ef9e1c5221e7f1f 100644
GIT binary patch
delta 7607
zcma)B3s@9amabd<;L(84ARq`7zPel52Eiwaii~emD6~NVvFR?LW7FMM6(T`xHBU7U
zGuMqKF^Sp9X3Q8TacgEZiP@Mn6IablFluy3KF93tOp@6h7xUQc<jZXKoT~1o8<W}T
z@8i_D_x|^sbI&<-?!Cp}?{%l|)u)xDr6x1*GmU%ZuJ`F<X}P5Qgym;s_I^H_e~i!J
zf6V9dlpn(%;Pd#u;q&>&`LX;#zJNbenj}x>3;80R<BR!m{CIu>KarorKS7h(>Eube
zfc>Yje0>hX^Er$!U9RREfM28X3tOw|9-t>OGYxB7vw0^i)a6?mXA-o6VfeC&M7{~I
z>k#WB)3k0z{!wJs#P!o6*r)hrK(>rPE{Y;sV~`sF*){^XIEvhemds=|WxNZp?TC$P
zUlK*S<B%Re3USD#QDjFPvJ;S9amZy+q&E(E7a%vqA(sR4AbPCW9bJ(RP~CB;+9=8!
zMfm|0h(oQ6Y7NF|-3&-E4!J6dl;V&wAbaAFtD~BwsOBwz>WxFyMNwO$s6Ig59fzuq
zqWYt#ZGhSyhvFlssXL;m5TJI(p&IyI(6<}=<Td;~^4iF(;(MdB?gPaAakHF&QuA}-
z199tq5RiM~kc|;}6Zgg;_W|;uIAl`<X?Zvf`3N8%jYF>E$#}+gSXrEqYOt1<b_KeH
z(z{(<o7|;83b=c^1-~qn2F1W#f*Y9LE|*;H36v^pQhs8wk0WWf86byi)Q4CD>Ff@_
zD{5w^0oB3Z7gi&ke~(S6owlANj1C$dOfZq|Dws%Xr{yYtm~Lg0Ao0gCys6HHSw|hS
zR_a$~Itjg$l4ayaIhhJVH_s~0Pv<jQN>s@}oQ8-`QyAV%kI!l(OK9no3Hn8BZ5A@L
zes*(CW=m;g!kW~`1U`$No$V&siluUoj!dAwstL-=RhcZQqG#tXBV(1+1z(b!uNWqy
zn!TBBKA8Mi@<4v|nPq3@UP-T`2@C7YMNC@GmmHIxxqrsK{(;22ujf$j!U8gmKCn;<
zRT$nHo3-4*yj7O6Ji+kR-0D1NoXxH_EzdW+J+3Aj9_MCPTi|h?GsDpN3e$3Z(uD-m
z^4z2g<|OF3kZ)REl5}BGBJvJXZHDf`Y;$dj?xJbpirEPl%aV|vZK_Q&y{k_I{@oM<
zVBFeDmA;z)(2?g9VM!ZHjw`=fHlC1^%4f^XEP0iFTl*w54z7GsM^4l7`d{X#0gD6;
zOS2lVcKY!L<LS>{P7B!+BL0J7sc(rkctZ?tYzdb#hAA~bIwgoz030+4TOfbhB-tg(
zgGpLL@OxGbnnahsQ-};`4HyvT!7Ljq8gR0{JrMBOXn8|^=q$7)l(>`mnjxf~+*axH
zT|?^g^sF<UH|A^nm36K#<KGfy`a>FA%TGOsiTu!Lb-A;E*~v8OW<k#!VhN9X(lING
zn9v$B+}Q~euyYOLWM{p^Fyk5VKER7UAU3`DzsP@wgaN0D1c-zxC&5pw24*lz^Fv^Z
zAo^TEsZCwIs0X0BOb#$Rnd_$POQvzxOmzdMx}(`i*Q^T$tP5UgKGS^3Qa7NlQ<p-y
z=dwwc8mJR|4sCwUM4z%Jr?(hEg8Hjq?RgACPwdVrcoRA_@9M=H$j?Wz8_5QSre7<Z
zat?TD5)kxp5;5OK^1S}}HGh{pgzQW5FKSL_zOwZA4j7@ueOU#lNA=zsuze>}$^?m%
zNoN*6%P>KfURuTJoGedIY#UAQd!!KCdT0}Rq9hAC3NrPL(`HpNjONc3E_k7J^sRMc
z^+5x?_yc`eMyFu|GmWLyYqLU)qtP;_uEhio^|yiWY-A6&r$%}~vu?#whDl-y87F~6
z)D0$hTzzeugg!_{zo#wSu#rJMu#(NF=9)MF0<cFG#Mgio8<61o006E_V+M`xKo68c
z5o1ILg@P&xhbM9P)8d-xcKJYsOH`GF;c4I+)w7QP8p4`7YUkn)Gjb@stGeccv`N>^
z1=q}$OJ>Vuv+co}>v`O>lb@J;q~U1s<vjb*%~$iv9$dU<#r2e&!?TYTU&*&$O|kD?
zaXXbU=MJSYDdwFkm4B_@MA|QdZ0TNA!PktPEQZ$u>4BL;>hpQG_B=iD%!D`W8h>Sd
z+dzBA)%6|W=9WH<`I2E4v&-0Mqls<B27S39FAZhF_$pn!4iwO94TbFEDazLk-zJu|
zOb6}i&zTV^vcWv(m~>`seh=$ldf3H0#3r8~<ILytocYwXp@`(uk6Ux-nbx8-jI%bK
z9}9!^e?OYO+cKK|*M<z;*kC@@wM{V;IL&+ko!(ZISqNM<CU)e<7tzkP>db<mndh7t
ze6h2DALks)k9X$Mw^~Q%Oo$l7Pi(;~Qh#-`X+}ogG=7qk@J2|>1gDO-IQ6{MY2a;6
zBR|=h&6hY6dAl>IG%Z-z0*M-)SVd3F$TTqg6lWn_x$z#IzV?^&^2S~8e4949roywE
z?sH8amG6aB7r|f9>}6~Y`kt$ba7t>sjU<Z+kQHJg5L)Z5Fo?;>J?!>P$in{7NV1XS
z02xe`1Rw0Zf~QS(bxL9`E%4-#B09@cXyg!7L|Z)DjF@L)1N}FTrKCvA#D*$sDi|%6
zBZ-$PeujOybh@ya*y&G&iaaq7C)to#fea?N1AbZ9DoaH)qhnl-mf{Al-y>`lYXI;*
z2$0g0l^yf+DIaA_II`w)#^jyLZ?j77rV^ISr;EBD4J9(87wG=<Im?)XI_V*paCTCK
zGo5*ROhu!a`Bh;xX&PgArzoo_EAgEQ7WjAPrZ!E{|2oZtw9trja#mBh{x>=!(i~gU
zO#OQXBU0PErn&k7yAkO7Db=~_i2m~A)OAMv<+*c#evm$=ag?s$S{i}JHS?UN4Bho~
zf;7tjdajQ#HBF{L|2CE-$}Rur24bck%U_ynarJe$dPC6Xatq>m`bkfvQGgyo6w8*^
zS#}YjAHO`F<SU7N_vvew;Dn_}_Mtx?MzRc>%aQEIW)YIdkgUKyRmDo=W@6te<W?i8
zqyMt~Z;6#|+rbeheR{`!gBQAC`wNwZoCuhL_<H0Tku)J$hh#mH;XpTIvlWOfS+i;<
zPCN=^FhLZg9v>93cEm(iD{jIjiW$^<WuaTbQ-D_4)apiWhVl#n7|v6^3%$dVO#065
z>13+%?e6DvaN>F9{@%(`l%Vb=4%CKkoH!i_90b&4+>9N4IP?Y*ETZCE?5m=y9~f`g
z3T&Cu`@lma3E?jiF-6^!$U>!L&k<5SEKjG&Q$?vMh#L^ts^+J-14SaKrhndh%Gi(1
zZFJPNA~+d-xbHXf`k-SlAt(x4yh86PIA{zDCaPcw08DtFPWB7EZBn2|ghaR(`WMqr
z9{P8ax*XW{8BtUPuTcK`;nSp4Mpe6z+>K+4fV@ZsjrhL!WpOtQe3MkF36o02d+5iH
z<{35Rl<dy~myX@<Fzf@^a%J`Y8WL-Iy>jZY%`90%Cm$#wd#Url0)QVpFyBA{UZ;F`
zfG5WLQL|=HlhW|`OqQ&t`wq=8?#2H7^xUBmnDfP<8HPj9(Wp3{m=-r@gYw{$=LnH$
z-Qg<rkgNsIAZF4h56>_>1(RBptB1#t^x@^JA=OlND3)ig;`Bd1J3VLErA94Qs<SrJ
zWk+Tho&`Yyy?VHe{^H14<I`xICc&>f{oH2#C^Yw*gi`f@B!S%pFO<YMd^2r$VV2=}
zz;rA7Untils3siOGG&YM>5EH<NnP+yQ1E_Qd2B4%OdF4t8jeCo58Zd{2H8gMdg)Ok
z<^ce{d#T)T4A_1;<M<h}n|^)#e@xmCj{lT?^YaqJ&w<@VC%-&M?pF-2c#Ui_s|5dH
zB1v)dmV2gPk#u?eRDZH&#hiARdsC+f#{}&61;onn9?$glit?$`ioM;zfGE5C@^*Dv
zhga}<`U5?(&+8W|D}q}&k4x$jJly#5ifQf&H|0*|S*HfK4$u1@;Tm+Ri|#$y0LI@q
zSuy%ewEGN@dAOa59Y}W5Nv8yIiavCzhP*-tPB~1c5cMjO*J#qKrG`tu-mg@>`p<;j
zm`D>|Z)V5jQP1m+DRwTrK0$0yH=%&YRdE)Jol_SO>64(df4lPI*LRuPI(qzW*&Fb4
zlB+||09G*&knNndjRiy-*Po1KMHc(GeohqR9ud${@l~LpQgR1^f`yyUHRIe?u99nZ
zI2^!J*M+RLVXUon4!50iyJUA4XKe$q5n;_Z06Oq5YLD9|xWuL2&MqJP$&%IWb4k)F
zSGN$hhu%7$AKK`c&>tPxzR~9B2#AYZ?k=l8;1Rg0c^p0+Dx%ga$qrHIhGTq;EZdwr
z^lBL*h>|t3^6;AP^lfN<tlK={>-YCW!G<1by+c>HutgPt21!<J#t=pIfh9PV!yXt{
zW2we4v^}=p>&J0err>vb1q5g+Ic{}3ehqM7WrG20d@>9>&Dsq2zD^+~_N_K_V!x_9
zqFy^FgLHo&<PjuCw<{QRcT5tY=VOw{0Y~jlxhuSoh%9h`1Xihw#nZZ@!|Q`1y;aqL
zs>2K<*;HFtgtlbJ6t58Sd8Dw-tstX6nwEAhEZNS<0omoN)SMcYfB`~nT-bda=k4G$
zosvU>?<pho7pwYh+z1alLct!X%i7P4bld-@z8f}mZ_p=Hh7I|FWzQR4G`BtMxyVyZ
zg%BqOdNI)GY%tyDbqm%qJ6B$2Q^TUh#SUh}A@F-5xy9w{QS+o<gH&n{u7sO9?3%xn
zJ0a<>Po7A~yYc;3wf=CjL_*AQnu#$n4o}x28S3JyFf>qjtl@#?vQ`HkgKO|K?3W#o
z@!eJ%y!h$?(OTLfm6UecISXg84f|{Aj4~UyZ5tOO173q_878;)$b!|<B?=uD6=H!$
zC6rXv#A0|SBLRUUjaC{pleHymr$^40v<1EHO@cTj{I)|ihT!2YG+d=)x@s^rN9L>c
z%3~zsUU9n`e|qfVm{4r#?21&);leVam|9}?LP50H=5Qm+t%^|9!f2%UXhuUhx}(m9
zMig3?$73C-<gbwv&Nj`5KUk=Scb|FT0vy6CS2*1xqr+CJam8|?GBNu>)XeZoZ8p_N
z+=zPNtqJrx0I>+aiQ)DkK*X$;Zb6b<odSeWJ@#k?GKA$BtBBeMMFdZw9M+AXOT-EX
z*`*D^Hw)K%hujy`hArVQv{S-K0`cwec0!VA*)+oS4bg2kMw$cWbjRltstf1nom>B~
z9Z+}RToVy0;jI2}O^nH*Iao~b0IqFAknl?Sxr1y7Q&A3=T)|Tl$!WEikH~L$LDltn
z!1n=3kYuQ^BOv%%`Fni6=!!wMDEM5mcMI&~-=l`I7xS~VSjT8bF;@Dg3suUI^Gis5
zuUGE!`oT7EYL6I$tK=5K*Gj?Vx85lNUaPR;h-)gNJ#@y!TQP@yb~3B1cz2VIT%sr6
zlM~Ou$qcS>nY3cSYr2UIIM+!<gjV02qFfm`PuLsUeY8?J`#vSc5AaYYD=k-65#u28
zAJSK@7MMRm7B6)oekg-$&kg$9t0jitLHBGr>Vt1$41gO~>9v&!<UzXqMoXT0npRK5
zyHJFBwZXd#TwuPqu_xxHkOemG`RzuF9|i|g!tsUgV6c;2L43P<5(Q4m0&Ws5?#QI$
zZ)O^9Lszvj=jKs;tPO&a^gE-Dd`i<l%`F<CdDx_TQ1Bn<l1~?w4c|GyS=il%`nDqZ
zM<kyhLEFS3Bv<IGpN>mc=Ogi;Q}=-*4-_bgpKaG=uNN=igtbU+0U2Zil6YNt>OTt!
znMPmxV}8bmFl|s@8*pzDRX=}3fB(mYA^e6brXf+kAJ!m;9}h&lJd1eA)o!SGn-uo}
z8Po>@k}TpyPQ;s;h&L?}Z%HCvb40wih{urNHAKXNx_A)@=7Nav6VZdBx)dLM`c`hT
z9To#!jP#81=B<6i0RNu(d?{T$v?TSbH2iY@RU&-L9m=MMhL&XD2fZ(J3|lh9B0JQg
X3=Lf&NjpeJ($3VO1$z4H&maCj(HKka

delta 4260
zcmZ9P3s_S}7RT@0n;Ra12;pf0fq(%cNYL=|fr41A-~&g!Dj<j<2#64F0=`9&RxMb;
zV_R!=Yg_w$t?Rb7w|?8NRTo#=s%@3lM{U~Gec0V^cfY#Y+S=`7-8lgwDSrO%nKQqc
zbLPz4xp1OW<@vjyP1A-3bL_{i>~A=*y;mDc^lzjdArrc3G~GpG=#w;-?xrJY57pCN
zYM^^)90eLr_gaJ8+0;lAXd*SyB$`Y|Q8P`UPeCxBO`e7V{*S6<`K8-netX2s`OBgy
z&E+u;$I(nm*uPAgSvhi9go(@G=z?4_)MpE%3u8FCD28hcYb03%Zq7YzXfCsqHN-wc
z7c+9nugD@FxpWv=#>nztk<)!-g<pQl7+Z;~FZv@svdWKK&d6#%a)ysw;YZpTS>s2}
z^pOrfvX+r6{m4fd30Ox{m2X9LjH>seihY#BM>Q~Nl^<2&i`wWHbu}ZK{K(lpve}Px
zGP1>woa2k!?29Zj%H>DR^-*pg)yk+gKWd(jTH~YEGHRV4RXRkCTkoSbFlwV8HJ@%`
zzRl=!Q`+ucFf_`v#W!jzBRc#>Eo7AU)S9>Xt-F(vkNJ^{hUPOr?niEC<PJYl8X{A7
z`jJmCvdfQLOi40l*{4(ngo;F?K*&p}BNB=BF303Rk@o==n+$|gs<S#tK%`Th!8rq=
zFWwAYS+R;GXEYyV<G47EYDz75W?*&R9QbQqjA;Z7t4JGsoj6Gxsz3-w!^aB}XXD}M
zn+C|9kV9<HG<KBm2tONEk2S;F6Ut+B71p7V3qyxS(g+wc(N3b2CnxR{l>8|=o}@!*
z!7QRz_7~hE>U$g)UdY4C(@J#)P8)MCjth(E$?4kAAE>{d3jsy(#0cghS9^kZIxezU
z&z;T;nPn4CPcGE6#+m3sO>uyDHmN9@JwBLF7{MOrLv*a`e1c|HX3%+?rdSnpeo7GY
zTnNw<#{^x_1){Cjl;jH+%({|1;X<yt#2oNpToAKAG;2z-H6LaMqMau)#y+qlC1(1+
z1@gQiJX*z*-zxUvWI~QB+e_GTeh>TS>}QSax%&n34)o7IuIQK4lJ?VVq^pz37VbW0
zBQwZdukOvg<9!~s4?Itp`$nBiAF!{KRrRl^y;@c~*lb%jV6Y<1;Aio^#u9$n!$v*7
znrA>0-@<>#arL}B(4`^FIaio3u&@wZX5_DgDr3ri4%AvXr(OyV<7T|baZVjnR+t1G
z9RV$`MDnHC(EFvHcNzx0VZ*%9&N#^+#pO-pxRnB)ostzCCmCStcahLt6|8o~OZw7W
zFg+PDF20h>8ob6UN&3*4U{6Ue#i!uYP%&Z1T}kLj=#!$74J9QU7sMGkX=E6;$k@tf
zaIO3d%91ovy=0^Y$><A4<A#E<<1v^49JOnNc*(fO9tjuisX~I&2tU}zibhFCjWDJr
zAtHfU(dqi3$m}DN2o*Jj5s6M6HA&$#NlK*2QUV<%#p0x~)a*OODHWKv_os9c7AcFS
zN`$H<o(4z)9WAM-MH1;4NlnwFXqqks(y>yIRqHfWO!f_&%Bm9;njx98SOM`TnS*{u
z4A>nj1abC4IOo{Po_3gBJB~d=U`uWG2xC2~LIV5m)YWqqE1a#(Ct8JHX(7>`fST%c
zRjVB9JQ_z+P1Rrnvg>SG3-PE}+lA_O!0NijQ9Sz~$^wcCMP%X$bvYUxcDJLZ%3ZzE
zC9C0=Iz35**m|Q{i%=*`uHUGMLURF2mjS2hQ`3@Uf*$XL<Ip`G#eZt~p96eYLkruX
zjs_d#E{@mBfjE%eOPs7{;t8-fH@O{a-L53K)Q|+X8^VI|8{fyqx_XqrRZ~=f*TW;W
z6n_yO1-*+4r|(;OO`CFE7yp?q<r7`Xq4k$_<98HYH;#V3>Di`N792BOHf9}bxoXVY
zQQYI&Iq!N%%z+87n64PouZE;QQF1qw(?#FaDzrI`=jRaE+_Jkpnu{tB{`(Ej>70VA
zn}t1-Vq0S6&W^M#9ml<AEF@)l;`<4aWwyZgZ9KC-njBglqWV~?K{cvT$3&Kgt1b#^
z)EG;7wCa+mMzu^SH>mp4)l9FX6~>k)s;&ivmYY@A43n8&w@zA`A;f<^maxa?<0h45
z3LYy#wTUc_Cr?uz1h2{)`TLxb>iR|`0ia)ZPgj7Yn~tRuosHFYhddV&*H2V0WuEUj
zW%>GddG!QDwt{I<0tr(JH$A4BKM6gPQMzzbcA?~Ba|%iiHWN^KQKq8L8>SGgT=Y#t
zs|W={%YTMD9j(yaF`C4~@s31N4A(k(#AU3@2&=YPZN&(U#et8YRe~}bWe&<*l!wP=
z9yaI0$J>li5}Ljh%2i%^Pp3_^Gi!vB*ZDqA2t0T^o1`e&+g}h!7<~4`nu#XlIxt)Y
zN(wepnOFh_&Zz?(>#*x2${8<-J}d0#N){WLX;zMPbrUvu{;m{~tJLf|L_#0VBgh28
zcN>Cb9E6esIlJFhH)GQYpFEX7CPQS;MdNDb@B}zzM_avP&0GsVEEOV?J-r5TEh8pD
zV(%@D_YFFVsnmLBC{V0>P7rk+&e;lgAvU~?nLh?uUCphs-QlvzYe2tOuO7$>Chpa<
zY!~g#5Zf6xRoSt(hzw6?hI03*7M{!k`@S@?8Fue0V0iz&slnSBj=SQz42=g<m4ate
zh<ZJ152i6k>E55qlX-CVK#qDd`aAa=Ok-oy59Wy7tUX&<c`(aw$O7f`Z$2QT4t5^O
zA67Jx747n&9C0@rL>27|NhIvyn!Q^xu<&wa`HNR^c+TPMn1?l25A2<{LJsISoFf98
zs0_3(WWv{n&!~Gbw}Bb#%C(nUR3k9k9|Nq8Hb;}&C3k{!ByCug9ngIwPu$0t70US|
z;|1P9p#4}fS*4g>n@L7{m--CO*~#QpK^}=)(}psFiA9hHGwFHVDjr}+1Dt*R25Etp
zj_p?CzO#UNZ;Th8XQmUH-{>Q2AoI;1Gy`4O{~XwqH1S1dwt-#okPXU|x9Zh=GErWA
z$4ub(iAPBr{O^P<N<sQ9CR6YeD%W}wff*+q<R~0HSws%Qos$`wqe%S@<u$OJvWo99
zbAz(%)H%YhQ$cE9Io}utoqZX~&AzP~<zH9!@``-DfaqjLW3{`!&9T6H@i)s86CvYz
zJ_w%M!;TOC9Z_PxSS676p!0@1@Le{yg`NIAH|y0`n0=2Z%{PA{>Q6EJz)d46J%0hB
zzJQ)O#qzhgM12wMOR(uyysjTj{QFT3#bWaEr*QOEn)o?$tKiF9KMo5`LY1GtniD`a
zLh;uXdhfP)YuJiYtoPOetu@g1_0D0ZEs~u!>Gtx}6|B<}GWZIzFQU6zK8HN}RfTOQ
zvs{tPN`V^<BV#qyt?uTk8i$*G0p+h)UkEF>>!bjejzq!Eo3YBI|Fo)xr<SQ)`&KQG
zFX2z$$0qzLp@*5S!FV?y^8U2UhsT*!58YLm##)q9DE~mgl;v9}=iu@CNnzgcC{|3T
zi@@rG@ygZvn}p~k@&z2Q2<0{t58v#PKT_&`GLo?v%A;P%7Sq%w8<D6)`55IA_Nit!
z+gHinP3Z^a=V|TU4|*=f#jl5qcdU$eq4$Q9@urgTZt<v`%`Uf$_lArYfQ&D+j4z{%
zZ<CBKgN(;s#<MNsIh6l^f+t1BE%w${_AaXt(<wqJgw4NbgA>^j8Ns<gdHR<wBC<bU
zzL^dw4`%XrLSX%anc?_r^S1)Wr`_Svywjk3`rrx)T1di!wuIhk<ezz+kX`==>kd4~

diff --git a/compose.yaml b/compose.yaml
index 1523060..66c7571 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -18,10 +18,21 @@ services:
     command: ["sh", "-c", "pip install --no-cache-dir pytest==8.4.2 pytest-cov==6.2.1 && python -m pytest tests/unit -vv -rA --tb=short --cov=app --cov-report=term-missing --results-file=/test-results/latest.txt"]
     volumes:
       - ./test-results:/test-results
+      - ./tests/fixtures/source_cache:/app/tests/fixtures/source_cache
     environment:
       DATABASE_PATH: /tmp/what2do-test.db
       PYTHONPATH: /app
     profiles: ["test"]
 
+  cache-sources:
+    build: .
+    working_dir: /app
+    command: ["python", "scripts/cache_source_documents.py", "--delay", "2"]
+    volumes:
+      - ./tests/fixtures/source_cache:/app/tests/fixtures/source_cache
+    environment:
+      PYTHONPATH: /app
+    profiles: ["test"]
+
 volumes:
   what2do-data:
diff --git a/pytest.ini b/pytest.ini
index 3a2bd06..fdc7405 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,6 +1,7 @@
 [pytest]
 testpaths = tests
-addopts = -ra -m "not network"
+addopts = -ra -m "not network and not cached_sources"
 pythonpath = .
 markers =
     network: live-network integration tests, excluded from normal runs
+    cached_sources: offline extraction tests using manually cached source documents
diff --git a/scripts/cache_source_documents.py b/scripts/cache_source_documents.py
new file mode 100644
index 0000000..61c7175
--- /dev/null
+++ b/scripts/cache_source_documents.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+import argparse
+import hashlib
+import json
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+from app.crawler import _http_client
+
+
+ROOT = Path(__file__).resolve().parents[1]
+MANIFEST_PATH = ROOT / "tests" / "fixtures" / "illawarra_sources.json"
+CACHE_DIR = ROOT / "tests" / "fixtures" / "source_cache"
+INDEX_PATH = CACHE_DIR / "index.json"
+
+
+def cache_key(source):
+    digest = hashlib.sha256(source["url"].encode()).hexdigest()[:12]
+    return f"{digest}.html"
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Manually cache each Illawarra source once for offline extraction tests.")
+    parser.add_argument("--delay", type=float, default=2.0, help="Seconds to wait between source requests (default: 2).")
+    parser.add_argument("--timeout", type=float, default=30.0, help="Per-request timeout in seconds (default: 30).")
+    parser.add_argument("--limit", type=int, help="Only fetch the first N sources, useful for a trial run.")
+    parser.add_argument("--force", action="store_true", help="Replace documents already present in the cache.")
+    args = parser.parse_args()
+
+    sources = json.loads(MANIFEST_PATH.read_text())
+    if args.limit is not None:
+        sources = sources[:args.limit]
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    previous = json.loads(INDEX_PATH.read_text()) if INDEX_PATH.exists() else {"sources": {}}
+    records = previous.get("sources", {})
+
+    with _http_client(timeout=args.timeout) as client:
+        for position, source in enumerate(sources, start=1):
+            filename = cache_key(source)
+            destination = CACHE_DIR / filename
+            existing = records.get(source["url"])
+            if destination.exists() and existing and not args.force:
+                print(f"[{position}/{len(sources)}] SKIP {source['name']} (already cached)")
+                continue
+            print(f"[{position}/{len(sources)}] GET  {source['name']} — {source['url']}")
+            record = {"name": source["name"], "url": source["url"], "mode": source["mode"], "filename": filename, "fetched_at": datetime.now(timezone.utc).isoformat()}
+            try:
+                response = client.get(source["url"])
+                record.update({"status_code": response.status_code, "content_type": response.headers.get("content-type", ""), "final_url": str(response.url), "bytes": len(response.content)})
+                if response.status_code < 400 and "html" in record["content_type"].lower():
+                    destination.write_text(response.text, encoding="utf-8")
+                    record["cached"] = True
+                    print(f"             SAVED {filename} ({record['bytes']} bytes)")
+                else:
+                    record["cached"] = False
+                    record["error"] = f"Expected successful HTML response, got {response.status_code} {record['content_type'] or 'unknown content type'}"
+                    destination.unlink(missing_ok=True)
+                    print(f"             ERROR {record['error']}")
+            except Exception as exc:
+                record.update({"cached": False, "error": f"{type(exc).__name__}: {exc}"})
+                destination.unlink(missing_ok=True)
+                print(f"             ERROR {record['error']}")
+            records[source["url"]] = record
+            INDEX_PATH.write_text(json.dumps({"generated_at": datetime.now(timezone.utc).isoformat(), "sources": records}, indent=2) + "\n")
+            if position < len(sources) and args.delay > 0:
+                time.sleep(args.delay)
+
+    cached = sum(bool(record.get("cached")) for record in records.values())
+    print(f"Cache complete: {cached}/{len(records)} documents available in {CACHE_DIR}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/conftest.py b/tests/conftest.py
index d192ce9..b1315c5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -4,8 +4,12 @@ from pathlib import Path
 import pytest
 
 
+EXTRACTION_RESULTS = []
+
+
 def pytest_addoption(parser):
     parser.addoption("--results-file", help="Write the human-readable terminal report to this file.")
+    parser.addoption("--extraction-results", help="Write cached source extraction counts to this file.")
 
 
 def pytest_configure(config):
@@ -14,6 +18,22 @@ def pytest_configure(config):
         config.pluginmanager.register(_ResultsFilePlugin(Path(results_file)), "results-file")
 
 
+def pytest_sessionfinish(session, exitstatus):
+    report_option = session.config.getoption("--extraction-results")
+    if not report_option or not EXTRACTION_RESULTS:
+        return
+    report_path = Path(report_option)
+    rows = sorted(EXTRACTION_RESULTS, key=lambda row: row["name"].casefold())
+    lines = ["What2Do cached source extraction results", "=" * 39, "", f"{'Source':40} {'Mode':12} {'Events':>6}  Status", "-" * 75]
+    for row in rows:
+        lines.append(f"{row['name'][:40]:40} {row['mode'][:12]:12} {row['events']:6}  {row['status'].upper()}")
+        if row["message"]:
+            lines.append(f"  {row['message']}")
+    lines.extend(("", f"Sources tested: {len(rows)}", f"Events found: {sum(row['events'] for row in rows)}", f"Successful extractions: {sum(row['status'] == 'passed' for row in rows)}", f"Failed extractions: {sum(row['status'] == 'failed' for row in rows)}"))
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    report_path.write_text("\n".join(lines) + "\n")
+
+
 class _ResultsFilePlugin:
     def __init__(self, path):
         self.path = path
@@ -36,6 +56,11 @@ class _ResultsFilePlugin:
         self.path.write_text("\n".join(lines) + "\n")
 
 
+@pytest.fixture
+def record_extraction_result():
+    return EXTRACTION_RESULTS.append
+
+
 @pytest.fixture
 def source():
     return {
diff --git a/tests/fixtures/source_cache/.gitkeep b/tests/fixtures/source_cache/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/tests/unit/test_cached_source_extraction.py b/tests/unit/test_cached_source_extraction.py
new file mode 100644
index 0000000..1c878cd
--- /dev/null
+++ b/tests/unit/test_cached_source_extraction.py
@@ -0,0 +1,41 @@
+import json
+from pathlib import Path
+
+import pytest
+
+from app.crawler import CrawlError, detect_rules
+
+
+CACHE_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "source_cache"
+INDEX_PATH = CACHE_DIR / "index.json"
+
+
+
+def cached_sources():
+    if not INDEX_PATH.exists():
+        return []
+    index = json.loads(INDEX_PATH.read_text())
+    return [record for record in index.get("sources", {}).values() if record.get("cached")]
+
+
+@pytest.mark.cached_sources
+@pytest.mark.parametrize("record", cached_sources(), ids=lambda record: record["name"])
+def test_automatic_extraction_from_cached_source(record, record_extraction_result):
+    document = CACHE_DIR / record["filename"]
+    result = {"name": record["name"], "url": record["url"], "mode": record["mode"], "document": record["filename"], "events": 0, "status": "failed", "message": ""}
+    try:
+        rules, events, message = detect_rules(document.read_text(encoding="utf-8", errors="replace"), record["url"])
+        result.update({"events": len(events), "status": "passed" if events else "failed", "message": message, "item_selector": rules.get("item_selector", "")})
+        assert events, "Automatic extraction returned no events"
+    except CrawlError as exc:
+        result["message"] = str(exc)
+        pytest.fail(str(exc))
+    finally:
+        record_extraction_result(result)
+
+
+@pytest.mark.cached_sources
+def test_source_cache_exists():
+    assert INDEX_PATH.exists(), "No source cache exists. Run: python scripts/cache_source_documents.py"
+    assert cached_sources(), "The source cache index contains no successful HTML documents."
+