Pillar 1 v1.1 (shipped earlier) flagged benign browsing bursts as beaconing: on short captures, grid periods longer than the observed span never complete a cycle, so significance saturates to 1 and the score sticks at exactly 0.5. v1.2 caps the grid at a third of the observed span, requires a minimum number of completed cycles, and excludes protocols that are periodic by nature (NTP, DNS, mDNS, ICMP). Validated against 300 seeds per scenario: false positives on bursty traffic drop from the 290-300/300 that v1.1 produced to 0/300, with every beacon-detection scenario scoring unchanged. Add Pillar 2 (persistent/long-lived sessions, with Zeek's fragmented TCP sessions rejoined after a 5-minute gap), Pillar 3 (DNS tunneling via Shannon-entropy information volume per subdomain per hour; the Jensen-Shannon divergence was measured and does not separate tunnels from benign CDN subdomains, so it stays in the evidence at zero weight instead of in the score), and Pillar 4 (deterministic noisy-OR fusion across the three pillars). The fusion formula was validated against 8 labeled real-traffic vectors (IoT-23, CTU-Normal) with tolerance <= 1e-4. A known limitation is kept, not hidden: a benign Philips HUE bulb's strict periodic polling and a 21.9h session score it above three real C2 hosts. No weight or threshold was tuned to mask this; it is documented in the architecture docs instead. Also fixes real bugs found along the way: ingest_dir's partition count summed file names instead of distinct dates, DuckDB errors were counted as bad rows, /api/alerts always returned an empty list regardless of the last analysis, and the dashboard still claimed every pillar was unimplemented. Restores four design corrections that were missing from docs/architecture.md (KL to Jensen-Shannon, the persistent-session detector, the JA4 licensing boundary, and deterministic reduction order), and catalogs the public datasets used for validation (IoT-23, CTU-Normal, Mendeley DTQBC) with their licenses and SHA-256 checksums under research/validation/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VAyr8d7HuAdUGB6aA5xmDD
212 lines
7.0 KiB
Rust
212 lines
7.0 KiB
Rust
use crate::error::Result;
|
|
use crate::pillar4::{self, FusionInput, HostScore};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::BTreeSet;
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
|
pub struct Alert {
|
|
pub id: String,
|
|
pub pillar: u8,
|
|
pub score: f64,
|
|
pub summary: String,
|
|
pub evidence: serde_json::Value,
|
|
pub created_at: String,
|
|
}
|
|
|
|
pub fn alert_orig_h(alert: &Alert) -> String {
|
|
if let Some(h) = alert.evidence.get("orig_h").and_then(|v| v.as_str()) {
|
|
if !h.is_empty() {
|
|
return h.to_string();
|
|
}
|
|
}
|
|
if let Some(rest) = alert.summary.strip_prefix("Beaconing detected: ") {
|
|
if let Some(h) = rest.split(':').next() {
|
|
return h.to_string();
|
|
}
|
|
}
|
|
if let Some(rest) = alert.summary.strip_prefix("Persistent session: ") {
|
|
if let Some(h) = rest.split(" -> ").next() {
|
|
return h.to_string();
|
|
}
|
|
}
|
|
if let Some(rest) = alert
|
|
.summary
|
|
.strip_prefix("High-information DNS subdomains: ")
|
|
{
|
|
if let Some(h) = rest.split(" -> ").next() {
|
|
return h.to_string();
|
|
}
|
|
}
|
|
String::new()
|
|
}
|
|
|
|
pub fn fuse(alerts: &[Alert]) -> Result<Vec<HostScore>> {
|
|
let mut hosts_set = BTreeSet::new();
|
|
for alert in alerts {
|
|
let h = alert_orig_h(alert);
|
|
if !h.is_empty() {
|
|
hosts_set.insert(h);
|
|
}
|
|
}
|
|
|
|
let mut host_scores = Vec::new();
|
|
|
|
for orig_h in hosts_set {
|
|
let mut m1 = 0.0_f64;
|
|
let mut alert_id_p1 = None;
|
|
let mut m2 = 0.0_f64;
|
|
let mut alert_id_p2 = None;
|
|
let mut m3 = 0.0_f64;
|
|
let mut alert_id_p3 = None;
|
|
|
|
for alert in alerts {
|
|
if alert_orig_h(alert) == orig_h {
|
|
match alert.pillar {
|
|
1 => {
|
|
if alert.score > m1 || (alert.score == m1 && alert_id_p1.is_none()) {
|
|
m1 = alert.score;
|
|
alert_id_p1 = Some(alert.id.clone());
|
|
} else if alert.score == m1 {
|
|
if let Some(prev) = &alert_id_p1 {
|
|
if alert.id < *prev {
|
|
alert_id_p1 = Some(alert.id.clone());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
2 => {
|
|
if alert.score > m2 || (alert.score == m2 && alert_id_p2.is_none()) {
|
|
m2 = alert.score;
|
|
alert_id_p2 = Some(alert.id.clone());
|
|
} else if alert.score == m2 {
|
|
if let Some(prev) = &alert_id_p2 {
|
|
if alert.id < *prev {
|
|
alert_id_p2 = Some(alert.id.clone());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
3 => {
|
|
if alert.score > m3 || (alert.score == m3 && alert_id_p3.is_none()) {
|
|
m3 = alert.score;
|
|
alert_id_p3 = Some(alert.id.clone());
|
|
} else if alert.score == m3 {
|
|
if let Some(prev) = &alert_id_p3 {
|
|
if alert.id < *prev {
|
|
alert_id_p3 = Some(alert.id.clone());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
let input = FusionInput {
|
|
m1,
|
|
m2,
|
|
m3,
|
|
alert_id_p1,
|
|
alert_id_p2,
|
|
alert_id_p3,
|
|
};
|
|
|
|
let hs = pillar4::score(&orig_h, &input)?;
|
|
host_scores.push(hs);
|
|
}
|
|
|
|
host_scores.sort_by(|a, b| {
|
|
b.score
|
|
.total_cmp(&a.score)
|
|
.then_with(|| a.orig_h.cmp(&b.orig_h))
|
|
});
|
|
|
|
Ok(host_scores)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
struct SplitMix64 {
|
|
state: u64,
|
|
}
|
|
|
|
impl SplitMix64 {
|
|
fn new(seed: u64) -> Self {
|
|
Self { state: seed }
|
|
}
|
|
fn next_u64(&mut self) -> u64 {
|
|
self.state = self.state.wrapping_add(0x9e3779b97f4a7c15);
|
|
let mut z = self.state;
|
|
z = (z ^ (z >> 30)).wrapping_mul(0xbf58476d1ce4e5b9);
|
|
z = (z ^ (z >> 27)).wrapping_mul(0x94d049bb133111eb);
|
|
z ^ (z >> 31)
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_permutation_invariance_and_evidence() {
|
|
let alerts = vec![
|
|
Alert {
|
|
id: "alert_p1_a".to_string(),
|
|
pillar: 1,
|
|
score: 0.85,
|
|
summary: "Beaconing detected: 10.0.0.1:any -> 1.1.1.1:443 (period ~14400s, score 0.850)".to_string(),
|
|
evidence: serde_json::json!({ "orig_h": "10.0.0.1" }),
|
|
created_at: "2026-10-31T12:00:00Z".to_string(),
|
|
},
|
|
Alert {
|
|
id: "alert_p2_a".to_string(),
|
|
pillar: 2,
|
|
score: 0.60,
|
|
summary: "Persistent session: 10.0.0.1 -> 1.1.1.1:443 (6.0h, score 0.600)".to_string(),
|
|
evidence: serde_json::json!({ "orig_h": "10.0.0.1" }),
|
|
created_at: "2026-10-31T12:00:00Z".to_string(),
|
|
},
|
|
Alert {
|
|
id: "alert_p3_a".to_string(),
|
|
pillar: 3,
|
|
score: 0.50,
|
|
summary: "High-information DNS subdomains: 10.0.0.1 -> *.example.com (50000 bits/h, 100 unique, score 0.500)".to_string(),
|
|
evidence: serde_json::json!({ "orig_h": "10.0.0.1" }),
|
|
created_at: "2026-10-31T12:00:00Z".to_string(),
|
|
},
|
|
Alert {
|
|
id: "alert_p1_b".to_string(),
|
|
pillar: 1,
|
|
score: 0.45,
|
|
summary: "Beaconing detected: 10.0.0.2:any -> 2.2.2.2:80 (period ~900s, score 0.450)".to_string(),
|
|
evidence: serde_json::json!({ "orig_h": "10.0.0.2" }),
|
|
created_at: "2026-10-31T12:00:00Z".to_string(),
|
|
},
|
|
];
|
|
|
|
let reference = fuse(&alerts).unwrap();
|
|
let ref_json = serde_json::to_vec(&reference).unwrap();
|
|
|
|
// Check that each term.alert_id exists in original alerts
|
|
for hs in &reference {
|
|
for term in &hs.terms {
|
|
if let Some(id) = &term.alert_id {
|
|
assert!(alerts.iter().any(|a| &a.id == id));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test 10 random permutations
|
|
let mut rng = SplitMix64::new(42);
|
|
for _ in 0..10 {
|
|
let mut permuted = alerts.clone();
|
|
for i in (1..permuted.len()).rev() {
|
|
let j = (rng.next_u64() as usize) % (i + 1);
|
|
permuted.swap(i, j);
|
|
}
|
|
let fused = fuse(&permuted).unwrap();
|
|
let fused_json = serde_json::to_vec(&fused).unwrap();
|
|
assert_eq!(ref_json, fused_json);
|
|
}
|
|
}
|
|
}
|