feat: implement Pillars 2-4 and fix a production false-positive in Pillar 1

Pillar 1 v1.1 (shipped earlier) flagged benign browsing bursts as
beaconing: on short captures, grid periods longer than the observed
span never complete a cycle, so significance saturates to 1 and the
score sticks at exactly 0.5. v1.2 caps the grid at a third of the
observed span, requires a minimum number of completed cycles, and
excludes protocols that are periodic by nature (NTP, DNS, mDNS,
ICMP). Validated against 300 seeds per scenario: false positives on
bursty traffic drop from the 290-300/300 that v1.1 produced to 0/300,
with every beacon-detection scenario scoring unchanged.

Add Pillar 2 (persistent/long-lived sessions, with Zeek's fragmented
TCP sessions rejoined after a 5-minute gap), Pillar 3 (DNS tunneling
via Shannon-entropy information volume per subdomain per hour; the
Jensen-Shannon divergence was measured and does not separate tunnels
from benign CDN subdomains, so it stays in the evidence at zero
weight instead of in the score), and Pillar 4 (deterministic
noisy-OR fusion across the three pillars).

The fusion formula was validated against 8 labeled real-traffic
vectors (IoT-23, CTU-Normal) with tolerance <= 1e-4. A known
limitation is kept, not hidden: a benign Philips HUE bulb's strict
periodic polling and a 21.9h session score it above three real C2
hosts. No weight or threshold was tuned to mask this; it is
documented in the architecture docs instead.

Also fixes real bugs found along the way: ingest_dir's partition
count summed file names instead of distinct dates, DuckDB errors
were counted as bad rows, /api/alerts always returned an empty list
regardless of the last analysis, and the dashboard still claimed
every pillar was unimplemented.

Restores four design corrections that were missing from
docs/architecture.md (KL to Jensen-Shannon, the persistent-session
detector, the JA4 licensing boundary, and deterministic reduction
order), and catalogs the public datasets used for validation
(IoT-23, CTU-Normal, Mendeley DTQBC) with their licenses and SHA-256
checksums under research/validation/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VAyr8d7HuAdUGB6aA5xmDD

crates/acaua-core/tests/fixtures/zeek_split/SOURCE.txt

@@ -0,0 +1,13 @@
+Synthetic test fixture for Zeek idle TCP split verification.
+
+Generated with:
+1. python3 research/validation/mkpcap_idle.py /tmp/zeek_split_gen/pcap/idle.pcap
+2. docker run --rm --user "$(id -u):$(id -g)" \
+     -v /tmp/zeek_split_gen/pcap:/pcap:ro -v /tmp/zeek_split_gen/out:/out -w /out \
+     zeek/zeek:9.0.1 zeek -C -r /pcap/idle.pcap
+
+Contains one TCP session (10.0.0.5:40000 -> 203.0.113.9:443) split by Zeek 9.0.1 across a 600s idle gap into two conn.log lines:
+- First line: ts=1793400000.000000, duration=1741.050000, history=ShADd
+- Second line: ts=1793402401.000000, duration=1800.050000, history=DdFfA (no 'S' in history, gap = 659.95s <= 3600s)
+
+Expected chained session: L = 4201.05s, parts = 2.

crates/acaua-core/tests/fixtures/zeek_split/conn.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2026-11-01-00-28-56
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents	ip_proto
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]	count
+1793400000.000000	Cr5nhu4rvbFFhLnlC7	10.0.0.5	40000	203.0.113.9	443	tcp	-	1741.050000	600	1200	S1	T	T	0	ShADd	32	1880	31	2440	-	6
+1793402401.000000	Cn4avD2d83MCmnH4xl	10.0.0.5	40000	203.0.113.9	443	tcp	-	1800.050000	600	1200	SF	T	T	0	DdFfA	32	1880	31	2440	-	6
+#close	2026-11-01-00-28-56

crates/acaua-core/tests/pillar3_e2e.rs

@@ -0,0 +1,179 @@
+//! End-to-end Pilar 3: a synthetic Zeek `dns.log` is ingested into DuckDB,
+//! read back through the 7-day window and scored by `pillar3`. The tunnel
+//! names are synthetic (SplitMix64 base32), never real captures.
+
+use acaua_core::{pillar3, storage::Storage};
+use std::fs;
+
+struct SplitMix64 {
+    state: u64,
+}
+
+impl SplitMix64 {
+    fn new(seed: u64) -> Self {
+        Self { state: seed }
+    }
+    fn next_u64(&mut self) -> u64 {
+        self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15);
+        let mut z = self.state;
+        z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+        z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+        z ^ (z >> 31)
+    }
+    fn below(&mut self, n: u64) -> u64 {
+        self.next_u64() % n
+    }
+}
+
+const HEADER: &str = "#separator \\x09\n\
+#set_separator\t,\n\
+#empty_field\t(empty)\n\
+#unset_field\t-\n\
+#path\tdns\n\
+#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\ttrans_id\trtt\tquery\tqclass\tqclass_name\tqtype\tqtype_name\trcode\trcode_name\tAA\tTC\tRD\tRA\tZ\tanswers\tTTLs\trejected\topcode\topcode_name\n";
+
+fn dns_line(ts: f64, uid: &str, host: &str, port: u16, query: &str) -> String {
+    format!(
+        "{ts:.6}\t{uid}\t{host}\t{port}\t10.0.2.3\t53\tudp\t1\t0.001\t{query}\t1\tC_INET\t16\tTXT\t0\tNOERROR\tF\tF\tT\tT\t0\t-\t-\tF\t0\tQUERY\n"
+    )
+}
+
+fn base32_name(rng: &mut SplitMix64) -> String {
+    const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567";
+    let labels = 2 + rng.below(2) as usize;
+    let mut parts = Vec::new();
+    for _ in 0..labels {
+        let len = 50 + rng.below(14) as usize;
+        parts.push(
+            (0..len)
+                .map(|_| ALPHABET[rng.below(32) as usize] as char)
+                .collect::<String>(),
+        );
+    }
+    parts.join(".")
+}
+
+fn write_dns_log(dir: &std::path::Path, lines: &[String]) {
+    let mut log = String::from(HEADER);
+    for line in lines {
+        log.push_str(line);
+    }
+    fs::write(dir.join("dns.log"), log).unwrap();
+}
+
+#[test]
+#[allow(clippy::inconsistent_digit_grouping)]
+fn synthetic_tunnel_is_ingested_and_alerted() {
+    let temp = tempfile::tempdir().unwrap();
+    let logs = temp.path().join("logs");
+    fs::create_dir_all(&logs).unwrap();
+    let store = Storage::open(temp.path().join("data")).unwrap();
+
+    let mut rng = SplitMix64::new(2026_10_31);
+    let mut lines = Vec::new();
+    for i in 0..100 {
+        lines.push(dns_line(
+            1_800_000_000.0 + i as f64 * 30.0,
+            &format!("C{i}"),
+            "10.66.66.66",
+            50000 + i,
+            &format!("{}.t.example-tunnel.net", base32_name(&mut rng)),
+        ));
+    }
+    write_dns_log(&logs, &lines);
+
+    let stats = store.ingest_dns_directory(&logs).unwrap();
+    assert_eq!(stats.files_seen, 1);
+    assert_eq!(stats.records_ingested, 100);
+
+    let queries = store.query_window_dns().unwrap();
+    assert_eq!(queries.len(), 100);
+
+    let scores = pillar3::score(&pillar3::ProtocolInput { queries }).unwrap();
+    assert_eq!(scores.len(), 1);
+    let score = &scores[0];
+    assert_eq!(score.orig_h, "10.66.66.66");
+    assert_eq!(score.base_domain, "example-tunnel.net");
+    assert_eq!(score.status, "ok");
+    assert!(
+        score.score >= 0.40,
+        "score {} bits {}",
+        score.score,
+        score.information_bits
+    );
+    assert_eq!(score.weights.information, 1.0);
+    assert_eq!(score.weights.jsd, 0.0);
+    assert_eq!(score.jsd_reference_groups, 0);
+    assert_eq!(score.jsd, 0.0);
+
+    let alerts = pillar3::alerts(&scores).unwrap();
+    assert_eq!(alerts.len(), 1);
+    assert_eq!(alerts[0].pillar, 3);
+    assert_eq!(alerts[0].id.len(), 16);
+    assert_eq!(alerts[0].score, score.score);
+    let evidence: serde_json::Value = alerts[0].evidence.clone();
+    assert_eq!(evidence["weights"]["jsd"].as_f64(), Some(0.0));
+    assert_eq!(evidence["weights"]["information"].as_f64(), Some(1.0));
+}
+
+#[test]
+fn synthetic_cloudfront_stays_below_the_alert_threshold() {
+    let temp = tempfile::tempdir().unwrap();
+    let logs = temp.path().join("logs");
+    fs::create_dir_all(&logs).unwrap();
+    let store = Storage::open(temp.path().join("data")).unwrap();
+
+    let mut rng = SplitMix64::new(7);
+    let mut lines = Vec::new();
+    const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789";
+    for i in 0..45 {
+        let name: String = (0..14)
+            .map(|_| ALPHABET[rng.below(ALPHABET.len() as u64) as usize] as char)
+            .collect();
+        lines.push(dns_line(
+            1_800_000_000.0 + i as f64 * 60.0,
+            &format!("C{i}"),
+            "10.0.2.15",
+            41000 + i,
+            &format!("{name}.cloudfront.net"),
+        ));
+    }
+    write_dns_log(&logs, &lines);
+    store.ingest_dns_directory(&logs).unwrap();
+
+    let queries = store.query_window_dns().unwrap();
+    let scores = pillar3::score(&pillar3::ProtocolInput { queries }).unwrap();
+    let alerts = pillar3::alerts(&scores).unwrap();
+    assert!(alerts.is_empty());
+    assert_eq!(scores.len(), 1);
+    assert_eq!(scores[0].score, 0.0);
+}
+
+#[test]
+fn analyze_dir_emits_a_pillar3_alert_without_any_conn_log() {
+    let temp = tempfile::tempdir().unwrap();
+    let logs = temp.path().join("logs");
+    let data = temp.path().join("data");
+    fs::create_dir_all(&logs).unwrap();
+
+    let mut rng = SplitMix64::new(99);
+    let mut lines = Vec::new();
+    for i in 0..100 {
+        lines.push(dns_line(
+            1_800_000_000.0 + i as f64 * 30.0,
+            &format!("C{i}"),
+            "10.66.66.68",
+            52000 + i,
+            &format!("{}.t.example-tunnel.net", base32_name(&mut rng)),
+        ));
+    }
+    write_dns_log(&logs, &lines);
+
+    let report = acaua_core::zeek::analyze_dir(&logs, &data).unwrap();
+    assert_eq!(report.alerts.len(), 1);
+    assert_eq!(report.alerts[0].pillar, 3);
+    assert!(report.alerts[0]
+        .summary
+        .contains("High-information DNS subdomains"));
+    assert_eq!(report.alerts[0].id.len(), 16);
+}

crates/acaua-core/tests/smoke.rs

@@ -4,6 +4,7 @@ use acaua_core::{doctor, fusion, pillar1, pillar2, pillar3, pillar4, storage, ze
 fn pillar1_returns_a_skeleton_score() {
     let input = pillar1::BeaconingInput {
         timestamps: vec![1.0, 2.0, 3.0],
+        proto: "tcp".to_string(),
     };
     let score = pillar1::score(&input).unwrap();
     assert_eq!(score.observations, 3);
@@ -13,33 +14,49 @@ fn pillar1_returns_a_skeleton_score() {
 #[test]
 fn pillar2_counts_long_connections_without_panicking() {
     let input = pillar2::SessionInput {
-        durations_secs: vec![1.0, 400.0],
-        bytes: vec![10, 20],
+        sessions: vec![
+            pillar2::SessionCandidate {
+                start_ts: 1000.0,
+                end_ts: 1001.0,
+                duration: 1.0,
+                parts: 1,
+                orig_bytes: 10,
+                resp_bytes: 20,
+            },
+            pillar2::SessionCandidate {
+                start_ts: 2000.0,
+                end_ts: 2400.0,
+                duration: 400.0,
+                parts: 1,
+                orig_bytes: 100,
+                resp_bytes: 200,
+            },
+        ],
+        failed_attempts: 0,
     };
     let score = pillar2::score(&input).unwrap();
-    assert_eq!(score.long_connections, 1);
-    assert_eq!(score.status, "skeleton");
+    assert_eq!(score.longest_session_seconds, 400.0);
+    assert_eq!(score.status, "ok");
 }
 
 #[test]
-fn pillar3_handles_an_empty_protocol_list() {
-    let score = pillar3::score(&pillar3::ProtocolInput::default()).unwrap();
-    assert_eq!(score.distinct_protocols, 0);
-    assert_eq!(score.status, "skeleton");
+fn pillar3_handles_empty_dns_input() {
+    let scores = pillar3::score(&pillar3::ProtocolInput::default()).unwrap();
+    assert!(scores.is_empty());
 }
 
 #[test]
 fn pillar4_fuses_three_inputs() {
-    let score = pillar4::score(&pillar4::FusionInput::default()).unwrap();
-    assert_eq!(score.inputs, 3);
-    assert_eq!(score.status, "skeleton");
+    let score = pillar4::score("10.0.0.1", &pillar4::FusionInput::default()).unwrap();
+    assert_eq!(score.terms.len(), 3);
+    assert_eq!(score.formula_version, "1.0");
+    assert_eq!(score.score, 0.0);
 }
 
 #[test]
 fn fusion_returns_no_alerts_yet() {
-    let report = fusion::fuse(&[0.1, 0.2]).unwrap();
-    assert!(report.alerts.is_empty());
-    assert_eq!(report.status, "skeleton");
+    let hosts = fusion::fuse(&[]).unwrap();
+    assert!(hosts.is_empty());
 }
 
 #[test]
@@ -67,5 +84,5 @@ fn zeek_classifies_known_logs() {
 fn doctor_reports_checks() {
     let report = doctor::run();
     assert!(!report.checks.is_empty());
-    assert_eq!(report.status, "skeleton");
+    assert!(report.status == "ok" || report.status == "degraded");
 }