feat: implement Pillars 2-4 and fix a production false-positive in Pillar 1
Pillar 1 v1.1 (shipped earlier) flagged benign browsing bursts as beaconing: on short captures, grid periods longer than the observed span never complete a cycle, so significance saturates to 1 and the score sticks at exactly 0.5. v1.2 caps the grid at a third of the observed span, requires a minimum number of completed cycles, and excludes protocols that are periodic by nature (NTP, DNS, mDNS, ICMP). Validated against 300 seeds per scenario: false positives on bursty traffic drop from the 290-300/300 that v1.1 produced to 0/300, with every beacon-detection scenario scoring unchanged. Add Pillar 2 (persistent/long-lived sessions, with Zeek's fragmented TCP sessions rejoined after a 5-minute gap), Pillar 3 (DNS tunneling via Shannon-entropy information volume per subdomain per hour; the Jensen-Shannon divergence was measured and does not separate tunnels from benign CDN subdomains, so it stays in the evidence at zero weight instead of in the score), and Pillar 4 (deterministic noisy-OR fusion across the three pillars). The fusion formula was validated against 8 labeled real-traffic vectors (IoT-23, CTU-Normal) with tolerance <= 1e-4. A known limitation is kept, not hidden: a benign Philips HUE bulb's strict periodic polling and a 21.9h session score it above three real C2 hosts. No weight or threshold was tuned to mask this; it is documented in the architecture docs instead. Also fixes real bugs found along the way: ingest_dir's partition count summed file names instead of distinct dates, DuckDB errors were counted as bad rows, /api/alerts always returned an empty list regardless of the last analysis, and the dashboard still claimed every pillar was unimplemented. Restores four design corrections that were missing from docs/architecture.md (KL to Jensen-Shannon, the persistent-session detector, the JA4 licensing boundary, and deterministic reduction order), and catalogs the public datasets used for validation (IoT-23, CTU-Normal, Mendeley DTQBC) with their licenses and SHA-256 checksums under research/validation/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VAyr8d7HuAdUGB6aA5xmDD
This commit is contained in:
Lucas RayanandClaude Sonnet 5 committed
1 parent
8a80504c45
commit
c64bbbf775
41 files changed
+4034
-348
No files matched your search
@@ -0,0 +1,454 @@
|
||||
//! `dns.log` ingestion for Pilar 3.
|
||||
//!
|
||||
//! The generic Zeek TSV header (`#separator`, `#unset_field`, `#empty_field`,
|
||||
//! `#fields`) is honored and columns are mapped by name, never by position.
|
||||
//! `query` is stored exactly as Zeek wrote it: `\xNN` escapes are **not**
|
||||
//! decoded, because the entropy is computed over that deterministic
|
||||
//! representation. `-` and `(empty)` become SQL NULL.
|
||||
|
||||
use crate::error::{AcauaError, Result};
|
||||
use crate::pillar3::DnsQuery;
|
||||
use crate::storage::Storage;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DnsRecord {
|
||||
pub ts: f64,
|
||||
pub uid: String,
|
||||
pub orig_h: String,
|
||||
pub orig_p: u16,
|
||||
pub resp_h: String,
|
||||
pub resp_p: u16,
|
||||
pub proto: String,
|
||||
pub query: Option<String>,
|
||||
pub qtype_name: Option<String>,
|
||||
pub rcode_name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, serde::Serialize)]
|
||||
pub struct DnsIngestionStats {
|
||||
pub files_seen: usize,
|
||||
pub records_ingested: u64,
|
||||
pub lines_skipped: u64,
|
||||
pub partitions_affected: usize,
|
||||
}
|
||||
|
||||
fn has_parquet(conn: &duckdb::Connection, glob: &str) -> bool {
|
||||
let query = format!("SELECT count(*) FROM read_parquet('{glob}', hive_partitioning = true);");
|
||||
conn.query_row(&query, [], |row| row.get::<_, i64>(0))
|
||||
.map(|count| count > 0)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
fn dns_glob(&self) -> String {
|
||||
self.root()
|
||||
.join("raw")
|
||||
.join("dns")
|
||||
.join("*")
|
||||
.join("*.parquet")
|
||||
.to_string_lossy()
|
||||
.replace('\'', "''")
|
||||
}
|
||||
|
||||
fn conn_glob_string(&self) -> String {
|
||||
self.root()
|
||||
.join("raw")
|
||||
.join("conn")
|
||||
.join("*")
|
||||
.join("*.parquet")
|
||||
.to_string_lossy()
|
||||
.replace('\'', "''")
|
||||
}
|
||||
|
||||
/// Parses a Zeek `dns.log`. Returns the records, the number of skipped
|
||||
/// lines and the set of UTC day partitions touched.
|
||||
pub fn parse_dns_lines(content: &[u8]) -> (Vec<DnsRecord>, u64, HashSet<String>) {
|
||||
let mut records = Vec::new();
|
||||
let mut skipped = 0u64;
|
||||
let mut dates = HashSet::new();
|
||||
|
||||
let mut separator = '\t';
|
||||
let mut unset_field = String::from("-");
|
||||
let mut empty_field = String::from("(empty)");
|
||||
let mut fields: Vec<String> = Vec::new();
|
||||
let mut idx: std::collections::HashMap<&'static str, usize> =
|
||||
std::collections::HashMap::new();
|
||||
|
||||
let text = String::from_utf8_lossy(content);
|
||||
|
||||
for line in text.lines() {
|
||||
let line = line.trim_end_matches(['\r', '\n']);
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if line.starts_with('#') {
|
||||
if let Some(rest) = line.strip_prefix("#separator ") {
|
||||
let value = rest.trim();
|
||||
if let Some(hex) = value.strip_prefix("\\x") {
|
||||
if let Ok(byte) = u8::from_str_radix(hex, 16) {
|
||||
separator = byte as char;
|
||||
}
|
||||
} else if let Some(c) = value.chars().next() {
|
||||
separator = c;
|
||||
}
|
||||
} else if let Some(rest) = line.strip_prefix("#unset_field") {
|
||||
unset_field = rest.trim().to_string();
|
||||
} else if let Some(rest) = line.strip_prefix("#empty_field") {
|
||||
empty_field = rest.trim().to_string();
|
||||
} else if let Some(rest) = line.strip_prefix("#fields") {
|
||||
let rest = rest.trim();
|
||||
fields = rest
|
||||
.split(separator)
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect();
|
||||
idx.clear();
|
||||
for (i, name) in fields.iter().enumerate() {
|
||||
match name.as_str() {
|
||||
"ts" => idx.insert("ts", i),
|
||||
"uid" => idx.insert("uid", i),
|
||||
"id.orig_h" => idx.insert("orig_h", i),
|
||||
"id.orig_p" => idx.insert("orig_p", i),
|
||||
"id.resp_h" => idx.insert("resp_h", i),
|
||||
"id.resp_p" => idx.insert("resp_p", i),
|
||||
"proto" => idx.insert("proto", i),
|
||||
"query" => idx.insert("query", i),
|
||||
"qtype_name" => idx.insert("qtype_name", i),
|
||||
"rcode_name" => idx.insert("rcode_name", i),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let required = ["ts", "orig_h", "orig_p", "resp_h", "resp_p", "query"];
|
||||
if required.iter().any(|name| !idx.contains_key(name)) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let parts: Vec<&str> = line.split(separator).collect();
|
||||
if parts.len() < fields.len() {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let value = |name: &str| idx.get(name).and_then(|&i| parts.get(i).copied());
|
||||
let text_value = |name: &str| -> Option<String> {
|
||||
let raw = value(name)?;
|
||||
if raw == unset_field || raw == "-" || raw == empty_field {
|
||||
None
|
||||
} else {
|
||||
Some(raw.to_string())
|
||||
}
|
||||
};
|
||||
|
||||
let Some(ts) = value("ts").and_then(|v| v.parse::<f64>().ok()) else {
|
||||
skipped += 1;
|
||||
continue;
|
||||
};
|
||||
let Some(orig_p) = value("orig_p").and_then(|v| v.parse::<u16>().ok()) else {
|
||||
skipped += 1;
|
||||
continue;
|
||||
};
|
||||
let Some(resp_p) = value("resp_p").and_then(|v| v.parse::<u16>().ok()) else {
|
||||
skipped += 1;
|
||||
continue;
|
||||
};
|
||||
|
||||
if let Some(day) = chrono::DateTime::from_timestamp(ts.floor() as i64, 0) {
|
||||
dates.insert(day.format("%Y-%m-%d").to_string());
|
||||
}
|
||||
|
||||
records.push(DnsRecord {
|
||||
ts,
|
||||
uid: text_value("uid").unwrap_or_default(),
|
||||
orig_h: text_value("orig_h").unwrap_or_default(),
|
||||
orig_p,
|
||||
resp_h: text_value("resp_h").unwrap_or_default(),
|
||||
resp_p,
|
||||
proto: text_value("proto").unwrap_or_default(),
|
||||
query: text_value("query"),
|
||||
qtype_name: text_value("qtype_name"),
|
||||
rcode_name: text_value("rcode_name"),
|
||||
});
|
||||
}
|
||||
|
||||
(records, skipped, dates)
|
||||
}
|
||||
|
||||
/// Ingests one `dns.log` into `raw/dns` (daily partitions, idempotent by
|
||||
/// content hash).
|
||||
pub fn ingest_dns_log(&self, path: impl AsRef<Path>) -> Result<(u64, u64, HashSet<String>)> {
|
||||
let path = path.as_ref();
|
||||
let content = fs::read(path)?;
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&content);
|
||||
let hash = hasher.finalize();
|
||||
let mut sha16 = String::with_capacity(16);
|
||||
for b in &hash[..8] {
|
||||
use std::fmt::Write;
|
||||
let _ = write!(sha16, "{b:02x}");
|
||||
}
|
||||
|
||||
let (records, skipped, dates) = Self::parse_dns_lines(&content);
|
||||
if records.is_empty() {
|
||||
return Ok((0, skipped, HashSet::new()));
|
||||
}
|
||||
|
||||
let dns_dir = self.root().join("raw").join("dns");
|
||||
fs::create_dir_all(&dns_dir)?;
|
||||
|
||||
let conn = self.open_connection()?;
|
||||
conn.execute_batch(
|
||||
"CREATE TEMP TABLE dns_rows (
|
||||
ts DOUBLE,
|
||||
uid VARCHAR,
|
||||
orig_h VARCHAR,
|
||||
orig_p USMALLINT,
|
||||
resp_h VARCHAR,
|
||||
resp_p USMALLINT,
|
||||
proto VARCHAR,
|
||||
query VARCHAR,
|
||||
qtype_name VARCHAR,
|
||||
rcode_name VARCHAR
|
||||
);",
|
||||
)?;
|
||||
|
||||
{
|
||||
let mut appender = conn.appender("dns_rows")?;
|
||||
for r in &records {
|
||||
appender.append_row(duckdb::params![
|
||||
r.ts,
|
||||
r.uid,
|
||||
r.orig_h,
|
||||
r.orig_p,
|
||||
r.resp_h,
|
||||
r.resp_p,
|
||||
r.proto,
|
||||
r.query,
|
||||
r.qtype_name,
|
||||
r.rcode_name
|
||||
])?;
|
||||
}
|
||||
}
|
||||
|
||||
let dns_dir_sql = dns_dir.to_string_lossy().replace('\'', "''");
|
||||
let copy_sql = format!(
|
||||
"COPY (
|
||||
SELECT *, CAST(to_timestamp(ts) AS DATE) AS date FROM dns_rows
|
||||
) TO '{dns_dir_sql}' (
|
||||
FORMAT parquet,
|
||||
PARTITION_BY (date),
|
||||
FILENAME_PATTERN 'src_{sha16}_{{i}}',
|
||||
OVERWRITE_OR_IGNORE
|
||||
);"
|
||||
);
|
||||
conn.execute_batch(©_sql)?;
|
||||
conn.execute_batch("DROP TABLE dns_rows;")?;
|
||||
|
||||
Ok((records.len() as u64, skipped, dates))
|
||||
}
|
||||
|
||||
/// Ingests every `dns.log` / `dns.*.log` directly inside `directory`.
|
||||
pub fn ingest_dns_directory(&self, directory: impl AsRef<Path>) -> Result<DnsIngestionStats> {
|
||||
let directory = directory.as_ref();
|
||||
if !directory.is_dir() {
|
||||
return Err(AcauaError::InvalidInput(format!(
|
||||
"{} is not a directory",
|
||||
directory.display()
|
||||
)));
|
||||
}
|
||||
let canonical_dir = fs::canonicalize(directory)?;
|
||||
let mut stats = DnsIngestionStats::default();
|
||||
let mut dates = HashSet::new();
|
||||
|
||||
for entry in fs::read_dir(&canonical_dir)?.flatten() {
|
||||
let path = entry.path();
|
||||
if let Ok(canon) = fs::canonicalize(&path) {
|
||||
if !canon.starts_with(&canonical_dir) {
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
if name.ends_with(".gz") {
|
||||
continue;
|
||||
}
|
||||
let is_dns = name == "dns.log" || (name.starts_with("dns.") && name.ends_with(".log"));
|
||||
if !is_dns || !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
stats.files_seen += 1;
|
||||
match self.ingest_dns_log(&path) {
|
||||
Ok((ingested, skipped, file_dates)) => {
|
||||
stats.records_ingested += ingested;
|
||||
stats.lines_skipped += skipped;
|
||||
dates.extend(file_dates);
|
||||
}
|
||||
Err(_) => {
|
||||
stats.lines_skipped += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stats.partitions_affected = dates.len();
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// Reads the DNS rows inside the same 7-day window used by the other
|
||||
/// pillars. The window is relative to the most recent day present in
|
||||
/// `conn` OR `dns`.
|
||||
pub fn query_window_dns(&self) -> Result<Vec<DnsQuery>> {
|
||||
let conn = self.open_connection()?;
|
||||
let dns_glob = self.dns_glob();
|
||||
if !has_parquet(&conn, &dns_glob) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut max_date: Option<String> = conn
|
||||
.query_row(
|
||||
&format!(
|
||||
"SELECT max(date)::VARCHAR FROM read_parquet('{dns_glob}', hive_partitioning = true);"
|
||||
),
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap_or(None);
|
||||
|
||||
let conn_glob = self.conn_glob_string();
|
||||
if has_parquet(&conn, &conn_glob) {
|
||||
let conn_max: Option<String> = conn
|
||||
.query_row(
|
||||
&format!(
|
||||
"SELECT max(date)::VARCHAR FROM read_parquet('{conn_glob}', hive_partitioning = true);"
|
||||
),
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap_or(None);
|
||||
max_date = match (max_date, conn_max) {
|
||||
(Some(a), Some(b)) => Some(a.max(b)),
|
||||
(a, b) => a.or(b),
|
||||
};
|
||||
}
|
||||
|
||||
let Some(max_date) = max_date.filter(|d| !d.is_empty()) else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
|
||||
let query = format!(
|
||||
"SELECT ts, orig_h, resp_p, query, qtype_name, rcode_name
|
||||
FROM read_parquet('{dns_glob}', hive_partitioning = true, union_by_name = true)
|
||||
WHERE query IS NOT NULL
|
||||
AND query <> ''
|
||||
AND date > CAST('{max_date}' AS DATE) - INTERVAL 7 DAY
|
||||
ORDER BY orig_h, ts, query;"
|
||||
);
|
||||
|
||||
let mut stmt = conn.prepare(&query)?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok(DnsQuery {
|
||||
ts: row.get(0)?,
|
||||
orig_h: row.get(1)?,
|
||||
resp_p: row.get(2)?,
|
||||
query: row.get(3)?,
|
||||
qtype_name: row.get(4)?,
|
||||
rcode_name: row.get(5)?,
|
||||
})
|
||||
})?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
out.push(row?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
// Header copied from a real Zeek 9.0.1 dns.log.
|
||||
const DNS_LOG: &str = "#separator \\x09\n\
|
||||
#set_separator\t,\n\
|
||||
#empty_field\t(empty)\n\
|
||||
#unset_field\t-\n\
|
||||
#path\tdns\n\
|
||||
#open\t2026-10-31-12-00-00\n\
|
||||
#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\ttrans_id\trtt\tquery\tqclass\tqclass_name\tqtype\tqtype_name\trcode\trcode_name\tAA\tTC\tRD\tRA\tZ\tanswers\tTTLs\trejected\topcode\topcode_name\n\
|
||||
#types\ttime\tstring\taddr\tport\taddr\tport\tenum\tcount\tinterval\tstring\tcount\tstring\tcount\tstring\tcount\tstring\tbool\tbool\tbool\tbool\tcount\tvector[string]\tvector[interval]\tbool\tcount\tstring\n\
|
||||
1800000000.000000\tC1\t10.0.2.15\t51000\t10.0.2.3\t53\tudp\t1\t0.001\taaa.bbb.example.com\t1\tC_INET\t1\tA\t0\tNOERROR\tF\tF\tT\tT\t0\t93.184.216.34\t60.000000\tF\t0\tQUERY\n\
|
||||
1800000030.000000\tC2\t10.0.2.15\t51001\t10.0.2.3\t53\tudp\t2\t0.001\t\\x01tunnel.example.com\t1\tC_INET\t16\tTXT\t0\tNOERROR\tF\tF\tT\tT\t0\t-\t-\tF\t0\tQUERY\n\
|
||||
1800000060.000000\tC3\t10.0.2.15\t51002\t10.0.2.3\t53\tudp\t3\t-\t-\t1\tC_INET\t1\tA\t3\tNXDOMAIN\tF\tF\tT\tT\t0\t-\t-\tF\t0\tQUERY\n\
|
||||
1800000090.000000\tC4\t10.0.2.15\t51003\t10.0.2.3\t53\tudp\t4\t0.001\t(empty)\t1\tC_INET\t1\tA\t3\tNXDOMAIN\tF\tF\tT\tT\t0\t-\t-\tF\t0\tQUERY\n";
|
||||
|
||||
#[test]
|
||||
fn dns_parser_maps_by_name_and_preserves_zeek_escapes() {
|
||||
let (records, skipped, dates) = Storage::parse_dns_lines(DNS_LOG.as_bytes());
|
||||
assert_eq!(skipped, 0);
|
||||
assert_eq!(records.len(), 4);
|
||||
assert_eq!(dates.len(), 1);
|
||||
|
||||
assert_eq!(records[0].query.as_deref(), Some("aaa.bbb.example.com"));
|
||||
assert_eq!(records[0].qtype_name.as_deref(), Some("A"));
|
||||
assert_eq!(records[0].rcode_name.as_deref(), Some("NOERROR"));
|
||||
|
||||
// `\x01` is kept literally, not decoded.
|
||||
assert_eq!(records[1].query.as_deref(), Some("\\x01tunnel.example.com"));
|
||||
|
||||
// `-` (unset) and `(empty)` become NULL.
|
||||
assert_eq!(records[2].query, None);
|
||||
assert_eq!(records[3].query, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dns_roundtrip_is_idempotent() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let store = Storage::open(temp.path()).unwrap();
|
||||
let log = temp.path().join("dns.log");
|
||||
fs::write(&log, DNS_LOG).unwrap();
|
||||
|
||||
let (first, skipped, partitions) = store.ingest_dns_log(&log).unwrap();
|
||||
assert_eq!(first, 4);
|
||||
assert_eq!(skipped, 0);
|
||||
assert_eq!(partitions.len(), 1);
|
||||
|
||||
let queries = store.query_window_dns().unwrap();
|
||||
assert_eq!(queries.len(), 2);
|
||||
assert_eq!(queries[0].orig_h, "10.0.2.15");
|
||||
|
||||
let (second, _, _) = store.ingest_dns_log(&log).unwrap();
|
||||
assert_eq!(second, 4);
|
||||
let queries = store.query_window_dns().unwrap();
|
||||
assert_eq!(queries.len(), 2, "re-ingest must not duplicate rows");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dns_directory_ingestion_counts_partitions() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let store = Storage::open(temp.path()).unwrap();
|
||||
let logs = temp.path().join("logs");
|
||||
fs::create_dir_all(&logs).unwrap();
|
||||
fs::write(logs.join("dns.log"), DNS_LOG).unwrap();
|
||||
fs::write(logs.join("dns.1.log"), DNS_LOG).unwrap();
|
||||
fs::write(logs.join("conn.log"), "not dns").unwrap();
|
||||
|
||||
let stats = store.ingest_dns_directory(&logs).unwrap();
|
||||
assert_eq!(stats.files_seen, 2);
|
||||
assert_eq!(stats.records_ingested, 8);
|
||||
assert_eq!(stats.partitions_affected, 1);
|
||||
}
|
||||
}
|
||||
@@ -93,8 +93,9 @@ pub fn run() -> DoctorReport {
|
||||
detail: "research workspace is isolated from the product".to_string(),
|
||||
},
|
||||
];
|
||||
let all_ok = checks.iter().all(|c| c.ok);
|
||||
DoctorReport {
|
||||
checks,
|
||||
status: "skeleton",
|
||||
status: if all_ok { "ok" } else { "degraded" },
|
||||
}
|
||||
}
|
||||
+196
-10
@@ -1,7 +1,9 @@
|
||||
use crate::error::Result;
|
||||
use crate::pillar4::{self, FusionInput, HostScore};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct Alert {
|
||||
pub id: String,
|
||||
pub pillar: u8,
|
||||
@@ -11,15 +13,199 @@ pub struct Alert {
|
||||
pub created_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct FusionReport {
|
||||
pub alerts: Vec<Alert>,
|
||||
pub status: &'static str,
|
||||
pub fn alert_orig_h(alert: &Alert) -> String {
|
||||
if let Some(h) = alert.evidence.get("orig_h").and_then(|v| v.as_str()) {
|
||||
if !h.is_empty() {
|
||||
return h.to_string();
|
||||
}
|
||||
}
|
||||
if let Some(rest) = alert.summary.strip_prefix("Beaconing detected: ") {
|
||||
if let Some(h) = rest.split(':').next() {
|
||||
return h.to_string();
|
||||
}
|
||||
}
|
||||
if let Some(rest) = alert.summary.strip_prefix("Persistent session: ") {
|
||||
if let Some(h) = rest.split(" -> ").next() {
|
||||
return h.to_string();
|
||||
}
|
||||
}
|
||||
if let Some(rest) = alert
|
||||
.summary
|
||||
.strip_prefix("High-information DNS subdomains: ")
|
||||
{
|
||||
if let Some(h) = rest.split(" -> ").next() {
|
||||
return h.to_string();
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
pub fn fuse(_scores: &[f64]) -> Result<FusionReport> {
|
||||
Ok(FusionReport {
|
||||
alerts: Vec::new(),
|
||||
status: "skeleton",
|
||||
})
|
||||
pub fn fuse(alerts: &[Alert]) -> Result<Vec<HostScore>> {
|
||||
let mut hosts_set = BTreeSet::new();
|
||||
for alert in alerts {
|
||||
let h = alert_orig_h(alert);
|
||||
if !h.is_empty() {
|
||||
hosts_set.insert(h);
|
||||
}
|
||||
}
|
||||
|
||||
let mut host_scores = Vec::new();
|
||||
|
||||
for orig_h in hosts_set {
|
||||
let mut m1 = 0.0_f64;
|
||||
let mut alert_id_p1 = None;
|
||||
let mut m2 = 0.0_f64;
|
||||
let mut alert_id_p2 = None;
|
||||
let mut m3 = 0.0_f64;
|
||||
let mut alert_id_p3 = None;
|
||||
|
||||
for alert in alerts {
|
||||
if alert_orig_h(alert) == orig_h {
|
||||
match alert.pillar {
|
||||
1 => {
|
||||
if alert.score > m1 || (alert.score == m1 && alert_id_p1.is_none()) {
|
||||
m1 = alert.score;
|
||||
alert_id_p1 = Some(alert.id.clone());
|
||||
} else if alert.score == m1 {
|
||||
if let Some(prev) = &alert_id_p1 {
|
||||
if alert.id < *prev {
|
||||
alert_id_p1 = Some(alert.id.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
2 => {
|
||||
if alert.score > m2 || (alert.score == m2 && alert_id_p2.is_none()) {
|
||||
m2 = alert.score;
|
||||
alert_id_p2 = Some(alert.id.clone());
|
||||
} else if alert.score == m2 {
|
||||
if let Some(prev) = &alert_id_p2 {
|
||||
if alert.id < *prev {
|
||||
alert_id_p2 = Some(alert.id.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
3 => {
|
||||
if alert.score > m3 || (alert.score == m3 && alert_id_p3.is_none()) {
|
||||
m3 = alert.score;
|
||||
alert_id_p3 = Some(alert.id.clone());
|
||||
} else if alert.score == m3 {
|
||||
if let Some(prev) = &alert_id_p3 {
|
||||
if alert.id < *prev {
|
||||
alert_id_p3 = Some(alert.id.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let input = FusionInput {
|
||||
m1,
|
||||
m2,
|
||||
m3,
|
||||
alert_id_p1,
|
||||
alert_id_p2,
|
||||
alert_id_p3,
|
||||
};
|
||||
|
||||
let hs = pillar4::score(&orig_h, &input)?;
|
||||
host_scores.push(hs);
|
||||
}
|
||||
|
||||
host_scores.sort_by(|a, b| {
|
||||
b.score
|
||||
.total_cmp(&a.score)
|
||||
.then_with(|| a.orig_h.cmp(&b.orig_h))
|
||||
});
|
||||
|
||||
Ok(host_scores)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
struct SplitMix64 {
|
||||
state: u64,
|
||||
}
|
||||
|
||||
impl SplitMix64 {
|
||||
fn new(seed: u64) -> Self {
|
||||
Self { state: seed }
|
||||
}
|
||||
fn next_u64(&mut self) -> u64 {
|
||||
self.state = self.state.wrapping_add(0x9e3779b97f4a7c15);
|
||||
let mut z = self.state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xbf58476d1ce4e5b9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94d049bb133111eb);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_permutation_invariance_and_evidence() {
|
||||
let alerts = vec![
|
||||
Alert {
|
||||
id: "alert_p1_a".to_string(),
|
||||
pillar: 1,
|
||||
score: 0.85,
|
||||
summary: "Beaconing detected: 10.0.0.1:any -> 1.1.1.1:443 (period ~14400s, score 0.850)".to_string(),
|
||||
evidence: serde_json::json!({ "orig_h": "10.0.0.1" }),
|
||||
created_at: "2026-10-31T12:00:00Z".to_string(),
|
||||
},
|
||||
Alert {
|
||||
id: "alert_p2_a".to_string(),
|
||||
pillar: 2,
|
||||
score: 0.60,
|
||||
summary: "Persistent session: 10.0.0.1 -> 1.1.1.1:443 (6.0h, score 0.600)".to_string(),
|
||||
evidence: serde_json::json!({ "orig_h": "10.0.0.1" }),
|
||||
created_at: "2026-10-31T12:00:00Z".to_string(),
|
||||
},
|
||||
Alert {
|
||||
id: "alert_p3_a".to_string(),
|
||||
pillar: 3,
|
||||
score: 0.50,
|
||||
summary: "High-information DNS subdomains: 10.0.0.1 -> *.example.com (50000 bits/h, 100 unique, score 0.500)".to_string(),
|
||||
evidence: serde_json::json!({ "orig_h": "10.0.0.1" }),
|
||||
created_at: "2026-10-31T12:00:00Z".to_string(),
|
||||
},
|
||||
Alert {
|
||||
id: "alert_p1_b".to_string(),
|
||||
pillar: 1,
|
||||
score: 0.45,
|
||||
summary: "Beaconing detected: 10.0.0.2:any -> 2.2.2.2:80 (period ~900s, score 0.450)".to_string(),
|
||||
evidence: serde_json::json!({ "orig_h": "10.0.0.2" }),
|
||||
created_at: "2026-10-31T12:00:00Z".to_string(),
|
||||
},
|
||||
];
|
||||
|
||||
let reference = fuse(&alerts).unwrap();
|
||||
let ref_json = serde_json::to_vec(&reference).unwrap();
|
||||
|
||||
// Check that each term.alert_id exists in original alerts
|
||||
for hs in &reference {
|
||||
for term in &hs.terms {
|
||||
if let Some(id) = &term.alert_id {
|
||||
assert!(alerts.iter().any(|a| &a.id == id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test 10 random permutations
|
||||
let mut rng = SplitMix64::new(42);
|
||||
for _ in 0..10 {
|
||||
let mut permuted = alerts.clone();
|
||||
for i in (1..permuted.len()).rev() {
|
||||
let j = (rng.next_u64() as usize) % (i + 1);
|
||||
permuted.swap(i, j);
|
||||
}
|
||||
let fused = fuse(&permuted).unwrap();
|
||||
let fused_json = serde_json::to_vec(&fused).unwrap();
|
||||
assert_eq!(ref_json, fused_json);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod dns_store;
|
||||
pub mod doctor;
|
||||
pub mod error;
|
||||
pub mod fusion;
|
||||
|
||||
@@ -4,6 +4,8 @@ use serde::{Deserialize, Serialize};
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct BeaconingInput {
|
||||
pub timestamps: Vec<f64>,
|
||||
#[serde(default)]
|
||||
pub proto: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
@@ -34,6 +36,13 @@ pub struct BeaconingScore {
|
||||
pub weights: WeightsInfo,
|
||||
pub formula_version: &'static str,
|
||||
pub status: &'static str,
|
||||
pub span_seconds: f64,
|
||||
pub min_cycles: usize,
|
||||
pub significance: f64,
|
||||
pub regularity: f64,
|
||||
pub significance_gate: f64,
|
||||
pub gate_regularity: f64,
|
||||
pub proto: String,
|
||||
}
|
||||
|
||||
fn quantile_cont(sorted: &[f64], q: f64) -> f64 {
|
||||
@@ -55,18 +64,24 @@ fn quantile_cont(sorted: &[f64], q: f64) -> f64 {
|
||||
}
|
||||
|
||||
pub fn score(input: &BeaconingInput) -> Result<BeaconingScore> {
|
||||
let grid_info = GridInfo {
|
||||
min_period_s: 600.0,
|
||||
max_period_s: 43200.0,
|
||||
points: 200,
|
||||
};
|
||||
let n = input.timestamps.len();
|
||||
let min_cycles = 3;
|
||||
let gate_regularity = 0.25;
|
||||
|
||||
let weights_info = WeightsInfo {
|
||||
significance: 0.5,
|
||||
regularity: 0.5,
|
||||
};
|
||||
|
||||
let n = input.timestamps.len();
|
||||
if n < 20 {
|
||||
let span = if n >= 2 {
|
||||
let mut sorted = input.timestamps.clone();
|
||||
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||||
sorted[n - 1] - sorted[0]
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
return Ok(BeaconingScore {
|
||||
score: 0.0,
|
||||
observations: n,
|
||||
@@ -77,16 +92,29 @@ pub fn score(input: &BeaconingInput) -> Result<BeaconingScore> {
|
||||
skew_bowley: 0.0,
|
||||
degenerate_quartiles: false,
|
||||
method: "rayleigh-schuster",
|
||||
grid: grid_info,
|
||||
grid: GridInfo {
|
||||
min_period_s: 600.0,
|
||||
max_period_s: 43200.0,
|
||||
points: 0,
|
||||
},
|
||||
weights: weights_info,
|
||||
formula_version: "1.1",
|
||||
formula_version: "1.2",
|
||||
status: "insufficient_data",
|
||||
span_seconds: span,
|
||||
min_cycles,
|
||||
significance: 0.0,
|
||||
regularity: 0.0,
|
||||
significance_gate: 0.0,
|
||||
gate_regularity,
|
||||
proto: input.proto.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut ts = input.timestamps.clone();
|
||||
ts.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
let span = ts[n - 1] - ts[0];
|
||||
|
||||
let mut deltas = Vec::with_capacity(n - 1);
|
||||
for i in 1..n {
|
||||
deltas.push(ts[i] - ts[i - 1]);
|
||||
@@ -110,21 +138,85 @@ pub fn score(input: &BeaconingInput) -> Result<BeaconingScore> {
|
||||
abs_devs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||||
let madm = quantile_cont(&abs_devs, 0.50);
|
||||
|
||||
let (madm_normalized, degenerate_deltas) = if q2 <= EPSILON {
|
||||
(0.0, true)
|
||||
} else {
|
||||
(madm / q2, false)
|
||||
};
|
||||
// Step 2: degenerate deltas check
|
||||
if q2 <= EPSILON {
|
||||
return Ok(BeaconingScore {
|
||||
score: 0.0,
|
||||
observations: n,
|
||||
period_detected_seconds: 0.0,
|
||||
p_value: 1.0,
|
||||
peak_power: 0.0,
|
||||
madm_normalized: 0.0,
|
||||
skew_bowley,
|
||||
degenerate_quartiles,
|
||||
method: "rayleigh-schuster",
|
||||
grid: GridInfo {
|
||||
min_period_s: 600.0,
|
||||
max_period_s: 0.0,
|
||||
points: 0,
|
||||
},
|
||||
weights: weights_info,
|
||||
formula_version: "1.2",
|
||||
status: "degenerate_deltas",
|
||||
span_seconds: span,
|
||||
min_cycles,
|
||||
significance: 0.0,
|
||||
regularity: 0.0,
|
||||
significance_gate: 0.0,
|
||||
gate_regularity,
|
||||
proto: input.proto.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
// Rayleigh/Schuster periodogram with M = 200 log-spaced periods from 600s to 43200s
|
||||
let t0 = ts[0];
|
||||
let num_periods = 200;
|
||||
let mut periods = Vec::with_capacity(num_periods);
|
||||
let mut powers = Vec::with_capacity(num_periods);
|
||||
let madm_normalized = madm / q2;
|
||||
|
||||
for j in 0..num_periods {
|
||||
// Step 3: Effective grid (periods <= span / min_cycles)
|
||||
let max_allowed_period = span / (min_cycles as f64);
|
||||
let num_grid_points = 200;
|
||||
let mut periods = Vec::new();
|
||||
for j in 0..num_grid_points {
|
||||
let period = 600.0 * 72.0_f64.powf(j as f64 / 199.0);
|
||||
periods.push(period);
|
||||
if period <= max_allowed_period {
|
||||
periods.push(period);
|
||||
}
|
||||
}
|
||||
|
||||
let m_eff = periods.len();
|
||||
if m_eff == 0 {
|
||||
let regularity = (1.0 - 2.0 * madm_normalized).clamp(0.0, 1.0);
|
||||
return Ok(BeaconingScore {
|
||||
score: 0.0,
|
||||
observations: n,
|
||||
period_detected_seconds: 0.0,
|
||||
p_value: 1.0,
|
||||
peak_power: 0.0,
|
||||
madm_normalized,
|
||||
skew_bowley,
|
||||
degenerate_quartiles,
|
||||
method: "rayleigh-schuster",
|
||||
grid: GridInfo {
|
||||
min_period_s: 600.0,
|
||||
max_period_s: 0.0,
|
||||
points: 0,
|
||||
},
|
||||
weights: weights_info,
|
||||
formula_version: "1.2",
|
||||
status: "insufficient_span",
|
||||
span_seconds: span,
|
||||
min_cycles,
|
||||
significance: 0.0,
|
||||
regularity,
|
||||
significance_gate: 0.0,
|
||||
gate_regularity,
|
||||
proto: input.proto.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
// Step 4: Rayleigh/Schuster periodogram on effective grid
|
||||
let t0 = ts[0];
|
||||
let mut powers = Vec::with_capacity(m_eff);
|
||||
|
||||
for &period in &periods {
|
||||
let f = 1.0 / period;
|
||||
let mut sum_cos = 0.0;
|
||||
let mut sum_sin = 0.0;
|
||||
@@ -144,35 +236,37 @@ pub fn score(input: &BeaconingInput) -> Result<BeaconingScore> {
|
||||
}
|
||||
}
|
||||
|
||||
// Harmonic resistance: largest period in the grid with P >= 0.5 * P_max
|
||||
// Step 9: Harmonic resistance: largest period in effective grid with P >= 0.5 * P_max
|
||||
let threshold_power = 0.5 * p_max;
|
||||
let mut detected_period = periods[0];
|
||||
for j in (0..num_periods).rev() {
|
||||
for j in (0..m_eff).rev() {
|
||||
if powers[j] >= threshold_power {
|
||||
detected_period = periods[j];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let m_points = num_periods as f64;
|
||||
let p_value = (m_points * (-p_max).exp()).min(1.0);
|
||||
let m_eff_f64 = m_eff as f64;
|
||||
let p_value = (m_eff_f64 * (-p_max).exp()).min(1.0);
|
||||
|
||||
// Formula v1.1
|
||||
let ln_m = m_points.ln();
|
||||
// Step 5: Significance
|
||||
let ln_m = m_eff_f64.ln();
|
||||
let six_ln_10 = 6.0 * 10.0_f64.ln();
|
||||
let sig = ((p_max - ln_m) / six_ln_10).clamp(0.0, 1.0);
|
||||
|
||||
// Step 6: Regularity
|
||||
let regularity = (1.0 - 2.0 * madm_normalized).clamp(0.0, 1.0);
|
||||
|
||||
let status = if degenerate_deltas {
|
||||
"degenerate_deltas"
|
||||
} else {
|
||||
"ok"
|
||||
};
|
||||
// Step 7: Gate
|
||||
let gate = (regularity / gate_regularity).min(1.0);
|
||||
|
||||
let score = if degenerate_deltas {
|
||||
0.5 * sig
|
||||
} else {
|
||||
0.5 * sig + 0.5 * regularity
|
||||
// Step 8: Score
|
||||
let score = 0.5 * sig * gate + 0.5 * regularity;
|
||||
|
||||
let grid_info = GridInfo {
|
||||
min_period_s: periods[0],
|
||||
max_period_s: periods[m_eff - 1],
|
||||
points: m_eff,
|
||||
};
|
||||
|
||||
Ok(BeaconingScore {
|
||||
@@ -187,8 +281,15 @@ pub fn score(input: &BeaconingInput) -> Result<BeaconingScore> {
|
||||
method: "rayleigh-schuster",
|
||||
grid: grid_info,
|
||||
weights: weights_info,
|
||||
formula_version: "1.1",
|
||||
status,
|
||||
formula_version: "1.2",
|
||||
status: "ok",
|
||||
span_seconds: span,
|
||||
min_cycles,
|
||||
significance: sig,
|
||||
regularity,
|
||||
significance_gate: gate,
|
||||
gate_regularity,
|
||||
proto: input.proto.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -233,7 +334,11 @@ mod tests {
|
||||
for k in 0..42 {
|
||||
timestamps.push(k as f64 * 14400.0);
|
||||
}
|
||||
let res = score(&BeaconingInput { timestamps }).unwrap();
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(res.score >= 0.99, "score: {}", res.score);
|
||||
let rel_err = (res.period_detected_seconds - 14400.0).abs() / 14400.0;
|
||||
assert!(
|
||||
@@ -243,6 +348,7 @@ mod tests {
|
||||
rel_err
|
||||
);
|
||||
assert_eq!(res.status, "ok");
|
||||
assert_eq!(res.formula_version, "1.2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -257,7 +363,11 @@ mod tests {
|
||||
let jitter = rng.uniform(-0.18, 0.18) * 14400.0;
|
||||
timestamps.push(t0 + k as f64 * 14400.0 + jitter);
|
||||
}
|
||||
let res = score(&BeaconingInput { timestamps }).unwrap();
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(res.score >= 0.80, "seed {seed} score {} < 0.80", res.score);
|
||||
if res.score < min_score {
|
||||
min_score = res.score;
|
||||
@@ -284,7 +394,11 @@ mod tests {
|
||||
curr += 14400.0 * (1.0 + rng.uniform(-0.18, 0.18));
|
||||
timestamps.push(curr);
|
||||
}
|
||||
let res = score(&BeaconingInput { timestamps }).unwrap();
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(res.score >= 0.40, "seed {seed} score {} < 0.40", res.score);
|
||||
if res.score < min_score {
|
||||
min_score = res.score;
|
||||
@@ -320,7 +434,11 @@ mod tests {
|
||||
curr += rng.exponential(mean);
|
||||
timestamps.push(curr);
|
||||
}
|
||||
let res = score(&BeaconingInput { timestamps }).unwrap();
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(res.score < 0.30, "seed {seed} score {} >= 0.30", res.score);
|
||||
if res.score < min_score {
|
||||
min_score = res.score;
|
||||
@@ -338,7 +456,11 @@ mod tests {
|
||||
for k in 0..25 {
|
||||
timestamps.push(k as f64 * 1000.0);
|
||||
}
|
||||
let res = score(&BeaconingInput { timestamps }).unwrap();
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(res.skew_bowley, 0.0);
|
||||
assert!(res.degenerate_quartiles);
|
||||
assert!(!res.score.is_nan() && !res.score.is_infinite());
|
||||
@@ -347,15 +469,24 @@ mod tests {
|
||||
#[test]
|
||||
fn test_6_median_delta_zero() {
|
||||
let timestamps = vec![1000.0; 25];
|
||||
let res = score(&BeaconingInput { timestamps }).unwrap();
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(res.status, "degenerate_deltas");
|
||||
assert_eq!(res.score, 0.0);
|
||||
assert!(!res.score.is_nan() && !res.score.is_infinite());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_7_insufficient_data() {
|
||||
let timestamps: Vec<f64> = (0..19).map(|k| k as f64 * 1000.0).collect();
|
||||
let res = score(&BeaconingInput { timestamps }).unwrap();
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(res.status, "insufficient_data");
|
||||
assert_eq!(res.score, 0.0);
|
||||
}
|
||||
@@ -365,11 +496,172 @@ mod tests {
|
||||
let timestamps: Vec<f64> = (0..42).map(|k| k as f64 * 14400.0).collect();
|
||||
let s1 = score(&BeaconingInput {
|
||||
timestamps: timestamps.clone(),
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
let s2 = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
let s2 = score(&BeaconingInput { timestamps }).unwrap();
|
||||
let j1 = serde_json::to_vec(&s1).unwrap();
|
||||
let j2 = serde_json::to_vec(&s2).unwrap();
|
||||
assert_eq!(j1, j2);
|
||||
}
|
||||
|
||||
fn generate_bursty(rng: &mut SplitMix64, span: f64) -> Vec<f64> {
|
||||
let mut out = Vec::new();
|
||||
let mut t = 1_790_000_000.0;
|
||||
let end = t + span;
|
||||
loop {
|
||||
t += rng.exponential(900.0);
|
||||
if t > end {
|
||||
break;
|
||||
}
|
||||
let mut k = 1;
|
||||
while rng.next_f64() < 0.8 {
|
||||
k += 1;
|
||||
}
|
||||
let mut u = t;
|
||||
for _ in 0..k {
|
||||
out.push(u);
|
||||
u += rng.exponential(2.0);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_9_bursty_3h() {
|
||||
let mut max_score = f64::MIN;
|
||||
for seed in 1..=50 {
|
||||
let mut rng = SplitMix64::new(seed);
|
||||
let timestamps = generate_bursty(&mut rng, 3.0 * 3600.0);
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
res.score < 0.05,
|
||||
"seed {seed} bursty3h score {} >= 0.05 (status: {})",
|
||||
res.score,
|
||||
res.status
|
||||
);
|
||||
if res.score > max_score {
|
||||
max_score = res.score;
|
||||
}
|
||||
}
|
||||
println!("Test 9 (bursty 3h) 50 seeds: max={max_score:.4}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_10_bursty_24h() {
|
||||
let mut max_score = f64::MIN;
|
||||
for seed in 1..=50 {
|
||||
let mut rng = SplitMix64::new(seed);
|
||||
let timestamps = generate_bursty(&mut rng, 86400.0);
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
res.score < 0.05,
|
||||
"seed {seed} bursty24h score {} >= 0.05 (status: {})",
|
||||
res.score,
|
||||
res.status
|
||||
);
|
||||
if res.score > max_score {
|
||||
max_score = res.score;
|
||||
}
|
||||
}
|
||||
println!("Test 10 (bursty 24h) 50 seeds: max={max_score:.4}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_11_fast_short() {
|
||||
let mut min_score = f64::MAX;
|
||||
let mut max_score = f64::MIN;
|
||||
let mut period_within_10pct = 0;
|
||||
|
||||
for seed in 1..=50 {
|
||||
let mut rng = SplitMix64::new(seed);
|
||||
let mut timestamps = Vec::new();
|
||||
let mut t = 1_790_000_000.0;
|
||||
let end = t + 86400.0;
|
||||
while t < end {
|
||||
timestamps.push(t);
|
||||
t += 900.0 * (1.0 + rng.uniform(-0.1, 0.1));
|
||||
}
|
||||
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
res.score >= 0.40,
|
||||
"seed {seed} fast_short score {} < 0.40",
|
||||
res.score
|
||||
);
|
||||
if res.score < min_score {
|
||||
min_score = res.score;
|
||||
}
|
||||
if res.score > max_score {
|
||||
max_score = res.score;
|
||||
}
|
||||
let rel_err = (res.period_detected_seconds - 900.0).abs() / 900.0;
|
||||
if rel_err < 0.10 {
|
||||
period_within_10pct += 1;
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"Test 11 (fast short) 50 seeds: min={min_score:.4}, max={max_score:.4}, period_hits={period_within_10pct}/50"
|
||||
);
|
||||
assert!(
|
||||
period_within_10pct >= 45,
|
||||
"period hits: {period_within_10pct}/50"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_12_dropout_30pct() {
|
||||
let mut hits = 0;
|
||||
for seed in 1..=50 {
|
||||
let mut rng = SplitMix64::new(seed);
|
||||
let mut timestamps = Vec::new();
|
||||
let mut t = 1_790_000_000.0;
|
||||
for _ in 0..60 {
|
||||
if rng.next_f64() > 0.3 {
|
||||
timestamps.push(t);
|
||||
}
|
||||
t += 14400.0 * (1.0 + rng.uniform(-0.1, 0.1));
|
||||
}
|
||||
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
if res.score >= 0.40 {
|
||||
hits += 1;
|
||||
}
|
||||
}
|
||||
println!("Test 12 (dropout 30%) 50 seeds: hits={hits}/50");
|
||||
assert!(hits >= 48, "dropout hits {hits} < 48");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_13_insufficient_span() {
|
||||
// 30 events in 25 min (1500s < 1800s)
|
||||
let timestamps: Vec<f64> = (0..30).map(|k| 1000.0 + k as f64 * 50.0).collect();
|
||||
let res = score(&BeaconingInput {
|
||||
timestamps,
|
||||
proto: "tcp".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(res.status, "insufficient_span");
|
||||
assert_eq!(res.score, 0.0);
|
||||
}
|
||||
}
|
||||
@@ -1,33 +1,309 @@
|
||||
use crate::error::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct SessionInput {
|
||||
pub durations_secs: Vec<f64>,
|
||||
pub bytes: Vec<u64>,
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct SessionCandidate {
|
||||
pub start_ts: f64,
|
||||
pub end_ts: f64,
|
||||
pub duration: f64,
|
||||
pub parts: usize,
|
||||
pub orig_bytes: u64,
|
||||
pub resp_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct SessionInput {
|
||||
pub sessions: Vec<SessionCandidate>,
|
||||
pub failed_attempts: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct ChainRuleInfo {
|
||||
pub max_gap_s: f64,
|
||||
pub continuation: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct SessionThresholds {
|
||||
pub zero_at_s: f64,
|
||||
pub saturate_at_s: f64,
|
||||
pub alert_at_s: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct SessionScore {
|
||||
pub score: f64,
|
||||
pub observations: usize,
|
||||
pub long_connections: usize,
|
||||
pub bowley_sentinel: Option<f64>,
|
||||
pub method: &'static str,
|
||||
pub status: &'static str,
|
||||
pub longest_session_seconds: f64,
|
||||
pub session_start_ts: f64,
|
||||
pub session_end_ts: f64,
|
||||
pub parts: usize,
|
||||
pub orig_bytes: u64,
|
||||
pub resp_bytes: u64,
|
||||
pub pcr: f64,
|
||||
pub bytes_per_hour: f64,
|
||||
pub failed_attempts: usize,
|
||||
pub sessions_over_1h: usize,
|
||||
pub method: &'static str,
|
||||
pub chain_rule: ChainRuleInfo,
|
||||
pub thresholds: SessionThresholds,
|
||||
pub formula_version: &'static str,
|
||||
}
|
||||
|
||||
pub fn score(input: &SessionInput) -> Result<SessionScore> {
|
||||
let chain_rule = ChainRuleInfo {
|
||||
max_gap_s: 3600.0,
|
||||
continuation: "history without 'S'",
|
||||
};
|
||||
let thresholds = SessionThresholds {
|
||||
zero_at_s: 3600.0,
|
||||
saturate_at_s: 86400.0,
|
||||
alert_at_s: 7200.0,
|
||||
};
|
||||
|
||||
let valid_sessions: Vec<&SessionCandidate> = input
|
||||
.sessions
|
||||
.iter()
|
||||
.filter(|s| s.duration > 0.0 && !s.duration.is_nan() && !s.duration.is_infinite())
|
||||
.collect();
|
||||
|
||||
if valid_sessions.is_empty() {
|
||||
return Ok(SessionScore {
|
||||
score: 0.0,
|
||||
status: "insufficient_data",
|
||||
longest_session_seconds: 0.0,
|
||||
session_start_ts: 0.0,
|
||||
session_end_ts: 0.0,
|
||||
parts: 0,
|
||||
orig_bytes: 0,
|
||||
resp_bytes: 0,
|
||||
pcr: 0.0,
|
||||
bytes_per_hour: 0.0,
|
||||
failed_attempts: input.failed_attempts,
|
||||
sessions_over_1h: 0,
|
||||
method: "chained-tcp-session-duration",
|
||||
chain_rule,
|
||||
thresholds,
|
||||
formula_version: "1.0",
|
||||
});
|
||||
}
|
||||
|
||||
let longest = valid_sessions
|
||||
.iter()
|
||||
.max_by(|a, b| a.duration.partial_cmp(&b.duration).unwrap())
|
||||
.unwrap();
|
||||
|
||||
let l = longest.duration;
|
||||
let persistence = if l > 3600.0 {
|
||||
((l / 3600.0).ln() / 24.0_f64.ln()).clamp(0.0, 1.0)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let tot_bytes = longest.orig_bytes + longest.resp_bytes;
|
||||
let pcr = if tot_bytes > 0 {
|
||||
(longest.orig_bytes as f64 - longest.resp_bytes as f64) / (tot_bytes as f64)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let bytes_per_hour = if l > 0.0 {
|
||||
(tot_bytes as f64) / (l / 3600.0)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let sessions_over_1h = input
|
||||
.sessions
|
||||
.iter()
|
||||
.filter(|s| s.duration >= 3600.0)
|
||||
.count();
|
||||
|
||||
Ok(SessionScore {
|
||||
score: 0.0,
|
||||
observations: input.durations_secs.len(),
|
||||
long_connections: input
|
||||
.durations_secs
|
||||
.iter()
|
||||
.filter(|duration| **duration >= 300.0)
|
||||
.count(),
|
||||
bowley_sentinel: None,
|
||||
method: "long-connection-stub",
|
||||
status: "skeleton",
|
||||
score: persistence,
|
||||
status: "ok",
|
||||
longest_session_seconds: l,
|
||||
session_start_ts: longest.start_ts,
|
||||
session_end_ts: longest.end_ts,
|
||||
parts: longest.parts,
|
||||
orig_bytes: longest.orig_bytes,
|
||||
resp_bytes: longest.resp_bytes,
|
||||
pcr,
|
||||
bytes_per_hour,
|
||||
failed_attempts: input.failed_attempts,
|
||||
sessions_over_1h,
|
||||
method: "chained-tcp-session-duration",
|
||||
chain_rule,
|
||||
thresholds,
|
||||
formula_version: "1.0",
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_1_formula_values() {
|
||||
let calc = |l: f64| -> f64 {
|
||||
let input = SessionInput {
|
||||
sessions: vec![SessionCandidate {
|
||||
start_ts: 1000.0,
|
||||
end_ts: 1000.0 + l,
|
||||
duration: l,
|
||||
parts: 1,
|
||||
orig_bytes: 100,
|
||||
resp_bytes: 100,
|
||||
}],
|
||||
failed_attempts: 0,
|
||||
};
|
||||
score(&input).unwrap().score
|
||||
};
|
||||
|
||||
assert_eq!(calc(3600.0), 0.0);
|
||||
let s7200 = calc(7200.0);
|
||||
assert!((s7200 - 0.218147).abs() < 1e-4, "s7200: {s7200}");
|
||||
assert_eq!(calc(86400.0), 1.0);
|
||||
assert_eq!(calc(200000.0), 1.0);
|
||||
assert_eq!(calc(0.0), 0.0);
|
||||
assert!(!calc(0.0).is_nan() && !calc(0.0).is_infinite());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_2_fixture_zeek_split_values() {
|
||||
// L = 4201.05s (between 1h and 2h)
|
||||
let input = SessionInput {
|
||||
sessions: vec![SessionCandidate {
|
||||
start_ts: 1793400000.0,
|
||||
end_ts: 1793402401.0 + 1800.05,
|
||||
duration: 4201.05,
|
||||
parts: 2,
|
||||
orig_bytes: 1200,
|
||||
resp_bytes: 2400,
|
||||
}],
|
||||
failed_attempts: 0,
|
||||
};
|
||||
let res = score(&input).unwrap();
|
||||
assert!((res.score - 0.0486).abs() < 1e-4, "score: {}", res.score);
|
||||
assert!(res.score < 0.2181, "should not trigger alert (< 2h)");
|
||||
assert_eq!(res.parts, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_3_c2_persistent_synthetic() {
|
||||
// 12.3h session in 9 parts + 13 failed attempts
|
||||
let duration = 12.3 * 3600.0;
|
||||
let input = SessionInput {
|
||||
sessions: vec![SessionCandidate {
|
||||
start_ts: 1000.0,
|
||||
end_ts: 1000.0 + duration,
|
||||
duration,
|
||||
parts: 9,
|
||||
orig_bytes: 19431,
|
||||
resp_bytes: 19431,
|
||||
}],
|
||||
failed_attempts: 13,
|
||||
};
|
||||
let res = score(&input).unwrap();
|
||||
assert!(res.score >= 0.78, "score: {}", res.score);
|
||||
assert_eq!(res.failed_attempts, 13);
|
||||
assert_eq!(res.parts, 9);
|
||||
assert_eq!(res.status, "ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_4_reverse_shell_synthetic() {
|
||||
// 2.5h session with PCR < 0 (more received than sent from victim perspective)
|
||||
let duration = 2.5 * 3600.0;
|
||||
let input = SessionInput {
|
||||
sessions: vec![SessionCandidate {
|
||||
start_ts: 1000.0,
|
||||
end_ts: 1000.0 + duration,
|
||||
duration,
|
||||
parts: 1,
|
||||
orig_bytes: 5000,
|
||||
resp_bytes: 50000,
|
||||
}],
|
||||
failed_attempts: 0,
|
||||
};
|
||||
let res = score(&input).unwrap();
|
||||
assert!(
|
||||
res.score >= 0.2181,
|
||||
"score should alert for 2.5h session: {}",
|
||||
res.score
|
||||
);
|
||||
assert!(res.pcr < 0.0, "PCR should be negative: {}", res.pcr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_5_web_browsing_synthetic() {
|
||||
// 200 HTTPS connections of 5s to 20 min in 3h
|
||||
let mut sessions = Vec::new();
|
||||
for i in 0..200 {
|
||||
let dur = 5.0 + (i as f64 % 1200.0);
|
||||
sessions.push(SessionCandidate {
|
||||
start_ts: 1000.0 + (i as f64 * 50.0),
|
||||
end_ts: 1000.0 + (i as f64 * 50.0) + dur,
|
||||
duration: dur,
|
||||
parts: 1,
|
||||
orig_bytes: 1000,
|
||||
resp_bytes: 10000,
|
||||
});
|
||||
}
|
||||
let input = SessionInput {
|
||||
sessions,
|
||||
failed_attempts: 0,
|
||||
};
|
||||
let res = score(&input).unwrap();
|
||||
assert_eq!(res.score, 0.0);
|
||||
assert!(res.score < 0.2181);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_6_ephemeral_port_reuse() {
|
||||
// Two distinct sessions with SYN on same 4-tuple, separated by 30 min
|
||||
let s1 = SessionCandidate {
|
||||
start_ts: 1000.0,
|
||||
end_ts: 2000.0,
|
||||
duration: 1000.0,
|
||||
parts: 1,
|
||||
orig_bytes: 100,
|
||||
resp_bytes: 100,
|
||||
};
|
||||
let s2 = SessionCandidate {
|
||||
start_ts: 3800.0,
|
||||
end_ts: 4800.0,
|
||||
duration: 1000.0,
|
||||
parts: 1,
|
||||
orig_bytes: 100,
|
||||
resp_bytes: 100,
|
||||
};
|
||||
let input = SessionInput {
|
||||
sessions: vec![s1, s2],
|
||||
failed_attempts: 0,
|
||||
};
|
||||
let res = score(&input).unwrap();
|
||||
assert_eq!(res.longest_session_seconds, 1000.0);
|
||||
assert_eq!(res.score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_7_determinism() {
|
||||
let input = SessionInput {
|
||||
sessions: vec![SessionCandidate {
|
||||
start_ts: 1793400000.123456,
|
||||
end_ts: 1793444000.654321,
|
||||
duration: 44000.530865,
|
||||
parts: 4,
|
||||
orig_bytes: 12345,
|
||||
resp_bytes: 67890,
|
||||
}],
|
||||
failed_attempts: 3,
|
||||
};
|
||||
let s1 = score(&input).unwrap();
|
||||
let s2 = score(&input).unwrap();
|
||||
let j1 = serde_json::to_vec(&s1).unwrap();
|
||||
let j2 = serde_json::to_vec(&s2).unwrap();
|
||||
assert_eq!(j1, j2);
|
||||
}
|
||||
}
|
||||
@@ -1,30 +1,671 @@
|
||||
use crate::error::Result;
|
||||
use crate::fusion::Alert;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// A single DNS query as read from the Zeek `dns.log`.
|
||||
///
|
||||
/// `query` is kept exactly as Zeek wrote it (including `\xNN` escapes); DNS
|
||||
/// names are not unescaped before the entropy calculation.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct DnsQuery {
|
||||
pub ts: f64,
|
||||
pub orig_h: String,
|
||||
pub resp_p: u16,
|
||||
pub query: String,
|
||||
pub qtype_name: Option<String>,
|
||||
pub rcode_name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ProtocolInput {
|
||||
pub events: usize,
|
||||
pub protocols: Vec<String>,
|
||||
pub queries: Vec<DnsQuery>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct Thresholds {
|
||||
pub zero_bits: f64,
|
||||
pub alert_bits: f64,
|
||||
pub saturate_bits: f64,
|
||||
pub min_unique: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct Weights {
|
||||
pub information: f64,
|
||||
pub jsd: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct ProtocolScore {
|
||||
pub score: f64,
|
||||
pub observations: usize,
|
||||
pub distinct_protocols: usize,
|
||||
pub method: &'static str,
|
||||
pub status: &'static str,
|
||||
pub orig_h: String,
|
||||
pub base_domain: String,
|
||||
pub peak_hour_start_ts: f64,
|
||||
pub unique_subdomains: usize,
|
||||
pub subdomain_chars: usize,
|
||||
pub entropy_bits_per_char: f64,
|
||||
pub information_bits: f64,
|
||||
pub mean_subdomain_len: f64,
|
||||
pub max_label_len: usize,
|
||||
/// Sorted by key; never a `HashMap`, so the evidence is deterministic.
|
||||
pub qtype_counts: BTreeMap<String, u64>,
|
||||
pub nxdomain_ratio: f64,
|
||||
pub jsd: f64,
|
||||
pub jsd_reference_groups: usize,
|
||||
pub method: &'static str,
|
||||
pub thresholds: Thresholds,
|
||||
pub weights: Weights,
|
||||
pub formula_version: &'static str,
|
||||
}
|
||||
|
||||
pub fn score(input: &ProtocolInput) -> Result<ProtocolScore> {
|
||||
let mut protocols = input.protocols.clone();
|
||||
protocols.sort();
|
||||
protocols.dedup();
|
||||
Ok(ProtocolScore {
|
||||
score: 0.0,
|
||||
observations: input.events,
|
||||
distinct_protocols: protocols.len(),
|
||||
method: "jensen-shannon-stub",
|
||||
status: "skeleton",
|
||||
const ZERO_BITS: f64 = 8192.0;
|
||||
const ALERT_BITS: f64 = 32768.0;
|
||||
const SATURATE_BITS: f64 = 262144.0;
|
||||
const MIN_UNIQUE: usize = 10;
|
||||
const ALERT_SCORE: f64 = 0.40;
|
||||
|
||||
const SLD2: [&str; 7] = ["co", "com", "net", "org", "gov", "edu", "ac"];
|
||||
|
||||
fn thresholds() -> Thresholds {
|
||||
Thresholds {
|
||||
zero_bits: ZERO_BITS,
|
||||
alert_bits: ALERT_BITS,
|
||||
saturate_bits: SATURATE_BITS,
|
||||
min_unique: MIN_UNIQUE,
|
||||
}
|
||||
}
|
||||
|
||||
fn weights() -> Weights {
|
||||
Weights {
|
||||
information: 1.0,
|
||||
jsd: 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalizes an ASCII DNS name (lowercase, trailing dot removed) and splits
|
||||
/// it into `(base_domain, subdomain)` with the spec heuristic (no Public
|
||||
/// Suffix List in v1). Returns `None` when there is no subdomain left.
|
||||
pub fn split_base_domain(query: &str) -> Option<(String, String)> {
|
||||
let normalized = query.trim_end_matches('.').to_ascii_lowercase();
|
||||
if normalized.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let labels: Vec<&str> = normalized.split('.').collect();
|
||||
if labels.iter().any(|label| label.is_empty()) {
|
||||
return None;
|
||||
}
|
||||
let k = if labels.len() >= 3
|
||||
&& labels[labels.len() - 1].len() == 2
|
||||
&& SLD2.contains(&labels[labels.len() - 2])
|
||||
{
|
||||
3
|
||||
} else {
|
||||
2
|
||||
};
|
||||
if labels.len() <= k {
|
||||
return None;
|
||||
}
|
||||
let base = labels[labels.len() - k..].join(".");
|
||||
let sub = labels[..labels.len() - k].join(".");
|
||||
if sub.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some((base, sub))
|
||||
}
|
||||
}
|
||||
|
||||
/// Full pre-filter and normalization for one `dns.log` row.
|
||||
///
|
||||
/// Excluded, with the reason documented in the spec:
|
||||
/// - `resp_p` 5353/5355/137: mDNS, LLMNR and NBNS are local discovery traffic.
|
||||
/// - bases ending in `in-addr.arpa`, `ip6.arpa` or `.local`: reverse and local
|
||||
/// names are structured and long by nature.
|
||||
pub fn classify_query(query: &str, resp_p: u16) -> Option<(String, String)> {
|
||||
if matches!(resp_p, 5353 | 5355 | 137) {
|
||||
return None;
|
||||
}
|
||||
let (base, sub) = split_base_domain(query)?;
|
||||
if base.ends_with("in-addr.arpa") || base.ends_with("ip6.arpa") || base.ends_with(".local") {
|
||||
return None;
|
||||
}
|
||||
Some((base, sub))
|
||||
}
|
||||
|
||||
/// Shannon entropy in bits per character over an aggregated character count.
|
||||
pub fn entropy_bits_per_char(counts: &BTreeMap<char, u64>) -> f64 {
|
||||
let total: u64 = counts.values().sum();
|
||||
if total == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let n = total as f64;
|
||||
let mut h = 0.0;
|
||||
for &count in counts.values() {
|
||||
let p = count as f64 / n;
|
||||
if p > 0.0 {
|
||||
h -= p * p.log2();
|
||||
}
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
/// Jensen-Shannon divergence (base 2) between two normalized distributions.
|
||||
/// Keys are visited in sorted order and `0 * log2(0)` terms are skipped, so
|
||||
/// the result is finite and symmetric bit-for-bit.
|
||||
pub fn jsd(p: &BTreeMap<char, f64>, q: &BTreeMap<char, f64>) -> f64 {
|
||||
let keys: BTreeSet<char> = p.keys().chain(q.keys()).copied().collect();
|
||||
let mut total = 0.0;
|
||||
for key in keys {
|
||||
let a = p.get(&key).copied().unwrap_or(0.0);
|
||||
let b = q.get(&key).copied().unwrap_or(0.0);
|
||||
let m = (a + b) / 2.0;
|
||||
let term_a = if a > 0.0 && m > 0.0 {
|
||||
0.5 * a * (a / m).log2()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let term_b = if b > 0.0 && m > 0.0 {
|
||||
0.5 * b * (b / m).log2()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
// Fixed summation order independent of the argument order: required
|
||||
// for JSD(P, Q) == JSD(Q, P) bit for bit.
|
||||
let (low, high) = if term_a <= term_b {
|
||||
(term_a, term_b)
|
||||
} else {
|
||||
(term_b, term_a)
|
||||
};
|
||||
total += low;
|
||||
total += high;
|
||||
}
|
||||
total
|
||||
}
|
||||
|
||||
fn normalized_counts(counts: &BTreeMap<char, u64>) -> BTreeMap<char, f64> {
|
||||
let total: u64 = counts.values().sum();
|
||||
if total == 0 {
|
||||
return BTreeMap::new();
|
||||
}
|
||||
let n = total as f64;
|
||||
counts.iter().map(|(&c, &v)| (c, v as f64 / n)).collect()
|
||||
}
|
||||
|
||||
/// Information volume in bits for one unique-subdomain set:
|
||||
/// `C` = characters without the dots, `H` = Shannon entropy over the
|
||||
/// aggregated character counts, `I = H * C`.
|
||||
fn information(subdomains: &BTreeSet<String>) -> (usize, usize, f64, f64) {
|
||||
let mut counts: BTreeMap<char, u64> = BTreeMap::new();
|
||||
let mut chars = 0usize;
|
||||
for sub in subdomains {
|
||||
for c in sub.chars().filter(|c| *c != '.') {
|
||||
*counts.entry(c).or_insert(0) += 1;
|
||||
chars += 1;
|
||||
}
|
||||
}
|
||||
let h = entropy_bits_per_char(&counts);
|
||||
(subdomains.len(), chars, h, h * chars as f64)
|
||||
}
|
||||
|
||||
fn score_from_bits(information_bits: f64) -> f64 {
|
||||
if information_bits <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
let raw = (information_bits / ZERO_BITS).log2() / (SATURATE_BITS / ZERO_BITS).log2();
|
||||
raw.clamp(0.0, 1.0)
|
||||
}
|
||||
|
||||
pub fn score(input: &ProtocolInput) -> Result<Vec<ProtocolScore>> {
|
||||
// (orig_h, base) -> hour -> unique subdomains
|
||||
let mut hourly: BTreeMap<(String, String), BTreeMap<i64, BTreeSet<String>>> = BTreeMap::new();
|
||||
// (orig_h, base) -> unique subdomains over the whole window (used by JSD)
|
||||
let mut window: BTreeMap<(String, String), BTreeSet<String>> = BTreeMap::new();
|
||||
// (orig_h, base) -> qtype counts and NXDOMAIN counters
|
||||
let mut qtypes: BTreeMap<(String, String), BTreeMap<String, u64>> = BTreeMap::new();
|
||||
let mut rcode_total: BTreeMap<(String, String), u64> = BTreeMap::new();
|
||||
let mut rcode_nx: BTreeMap<(String, String), u64> = BTreeMap::new();
|
||||
|
||||
for query in &input.queries {
|
||||
if query.query == "-" || query.query == "(empty)" || query.query.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Some((base, sub)) = classify_query(&query.query, query.resp_p) else {
|
||||
continue;
|
||||
};
|
||||
let key = (query.orig_h.clone(), base.clone());
|
||||
let hour = (query.ts / 3600.0).floor() as i64;
|
||||
hourly
|
||||
.entry(key.clone())
|
||||
.or_default()
|
||||
.entry(hour)
|
||||
.or_default()
|
||||
.insert(sub.clone());
|
||||
window.entry(key.clone()).or_default().insert(sub);
|
||||
|
||||
let qtype = query.qtype_name.clone().unwrap_or_else(|| "-".to_string());
|
||||
*qtypes
|
||||
.entry(key.clone())
|
||||
.or_default()
|
||||
.entry(qtype)
|
||||
.or_insert(0) += 1;
|
||||
*rcode_total.entry(key.clone()).or_insert(0) += 1;
|
||||
if query.rcode_name.as_deref() == Some("NXDOMAIN") {
|
||||
*rcode_nx.entry(key).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// JSD reference: equal-weight mixture of every OTHER (orig_h, base) group
|
||||
// in the window. It is evidence only (weight 0.0) and never enters score.
|
||||
let window_counts: BTreeMap<(String, String), BTreeMap<char, u64>> = window
|
||||
.iter()
|
||||
.map(|(key, subs)| {
|
||||
let mut counts: BTreeMap<char, u64> = BTreeMap::new();
|
||||
for sub in subs {
|
||||
for c in sub.chars().filter(|c| *c != '.') {
|
||||
*counts.entry(c).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
(key.clone(), counts)
|
||||
})
|
||||
.collect();
|
||||
let distributions: BTreeMap<(String, String), BTreeMap<char, f64>> = window_counts
|
||||
.iter()
|
||||
.map(|(key, counts)| (key.clone(), normalized_counts(counts)))
|
||||
.collect();
|
||||
let group_count = distributions.len();
|
||||
|
||||
let mut results = Vec::with_capacity(hourly.len());
|
||||
for ((orig_h, base_domain), hours) in &hourly {
|
||||
let mut best_bits = -1.0f64;
|
||||
let mut best_hour = 0i64;
|
||||
let mut best_unique = 0usize;
|
||||
let mut best_chars = 0usize;
|
||||
let mut best_entropy = 0.0f64;
|
||||
for (&hour, subs) in hours {
|
||||
let (unique, chars, entropy, bits) = information(subs);
|
||||
if bits > best_bits {
|
||||
best_bits = bits;
|
||||
best_hour = hour;
|
||||
best_unique = unique;
|
||||
best_chars = chars;
|
||||
best_entropy = entropy;
|
||||
}
|
||||
}
|
||||
let information_bits = best_bits.max(0.0);
|
||||
let key = (orig_h.clone(), base_domain.clone());
|
||||
let window_subs = &window[&key];
|
||||
|
||||
let mean_subdomain_len = if window_subs.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
let total: usize = window_subs.iter().map(|s| s.len()).sum();
|
||||
total as f64 / window_subs.len() as f64
|
||||
};
|
||||
let max_label_len = window_subs
|
||||
.iter()
|
||||
.flat_map(|s| s.split('.'))
|
||||
.map(|label| label.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
|
||||
let (jsd_value, jsd_reference_groups) = if group_count > 1 {
|
||||
let own = &distributions[&key];
|
||||
let divisor = (group_count - 1) as f64;
|
||||
let mut reference: BTreeMap<char, f64> = BTreeMap::new();
|
||||
for (other_key, dist) in &distributions {
|
||||
if other_key == &key {
|
||||
continue;
|
||||
}
|
||||
for (&c, &v) in dist {
|
||||
*reference.entry(c).or_insert(0.0) += v / divisor;
|
||||
}
|
||||
}
|
||||
reference.retain(|_, v| *v > 1e-15);
|
||||
(jsd(own, &reference), group_count - 1)
|
||||
} else {
|
||||
(0.0, 0)
|
||||
};
|
||||
|
||||
let total_rcode = *rcode_total.get(&key).unwrap_or(&0);
|
||||
let nxdomain_ratio = if total_rcode == 0 {
|
||||
0.0
|
||||
} else {
|
||||
*rcode_nx.get(&key).unwrap_or(&0) as f64 / total_rcode as f64
|
||||
};
|
||||
|
||||
let status = if best_unique < MIN_UNIQUE {
|
||||
"insufficient_data"
|
||||
} else {
|
||||
"ok"
|
||||
};
|
||||
let score = if best_unique < MIN_UNIQUE {
|
||||
0.0
|
||||
} else {
|
||||
score_from_bits(information_bits)
|
||||
};
|
||||
|
||||
results.push(ProtocolScore {
|
||||
score,
|
||||
status,
|
||||
orig_h: orig_h.clone(),
|
||||
base_domain: base_domain.clone(),
|
||||
peak_hour_start_ts: best_hour as f64 * 3600.0,
|
||||
unique_subdomains: best_unique,
|
||||
subdomain_chars: best_chars,
|
||||
entropy_bits_per_char: best_entropy,
|
||||
information_bits,
|
||||
mean_subdomain_len,
|
||||
max_label_len,
|
||||
qtype_counts: qtypes.get(&key).cloned().unwrap_or_default(),
|
||||
nxdomain_ratio,
|
||||
jsd: jsd_value,
|
||||
jsd_reference_groups,
|
||||
method: "dns-subdomain-information",
|
||||
thresholds: thresholds(),
|
||||
weights: weights(),
|
||||
formula_version: "1.0",
|
||||
});
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Stable `Alert.id`: SHA-256 of `3|orig_h|base_domain|peak_hour_start_ts`,
|
||||
/// truncated to 16 hex characters.
|
||||
pub fn alert_id(score: &ProtocolScore) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(
|
||||
format!(
|
||||
"3|{}|{}|{:.6}",
|
||||
score.orig_h, score.base_domain, score.peak_hour_start_ts
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
let hash = hasher.finalize();
|
||||
let mut id = String::with_capacity(16);
|
||||
for b in &hash[..8] {
|
||||
use std::fmt::Write;
|
||||
let _ = write!(id, "{b:02x}");
|
||||
}
|
||||
id
|
||||
}
|
||||
|
||||
/// Builds the alert for a `ProtocolScore` that reached the 0.40 threshold.
|
||||
/// JSD travels in the evidence with weight 0.0; it never changes the score.
|
||||
pub fn alert_for(score: &ProtocolScore) -> Result<Alert> {
|
||||
let summary = format!(
|
||||
"High-information DNS subdomains: {} -> *.{} ({} bits/h, {} unique, score {:.3})",
|
||||
score.orig_h,
|
||||
score.base_domain,
|
||||
score.information_bits.round() as u64,
|
||||
score.unique_subdomains,
|
||||
score.score
|
||||
);
|
||||
Ok(Alert {
|
||||
id: alert_id(score),
|
||||
pillar: 3,
|
||||
score: score.score,
|
||||
summary,
|
||||
evidence: serde_json::to_value(score)?,
|
||||
created_at: OffsetDateTime::now_utc().to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn alerts(scores: &[ProtocolScore]) -> Result<Vec<Alert>> {
|
||||
let mut out = Vec::new();
|
||||
for score in scores {
|
||||
if score.status == "ok" && score.score >= ALERT_SCORE {
|
||||
out.push(alert_for(score)?);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
struct SplitMix64 {
|
||||
state: u64,
|
||||
}
|
||||
|
||||
impl SplitMix64 {
|
||||
fn new(seed: u64) -> Self {
|
||||
Self { state: seed }
|
||||
}
|
||||
fn next_u64(&mut self) -> u64 {
|
||||
self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = self.state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
fn below(&mut self, n: u64) -> u64 {
|
||||
self.next_u64() % n
|
||||
}
|
||||
}
|
||||
|
||||
fn base32_char(rng: &mut SplitMix64) -> char {
|
||||
const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567";
|
||||
ALPHABET[rng.below(ALPHABET.len() as u64) as usize] as char
|
||||
}
|
||||
|
||||
fn hex_char(rng: &mut SplitMix64) -> char {
|
||||
const ALPHABET: &[u8] = b"0123456789abcdef";
|
||||
ALPHABET[rng.below(ALPHABET.len() as u64) as usize] as char
|
||||
}
|
||||
|
||||
fn base32_label(rng: &mut SplitMix64) -> String {
|
||||
let len = 50 + rng.below(14) as usize;
|
||||
(0..len).map(|_| base32_char(rng)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entropy_of_a_repeated_character_is_zero() {
|
||||
let mut counts = BTreeMap::new();
|
||||
counts.insert('a', 4);
|
||||
assert_eq!(entropy_bits_per_char(&counts), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entropy_of_uniform_32_symbols_is_five_bits() {
|
||||
let mut counts = BTreeMap::new();
|
||||
for i in 0..32u8 {
|
||||
counts.insert((b'a' + i) as char, 1000);
|
||||
}
|
||||
assert!((entropy_bits_per_char(&counts) - 5.0).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jsd_equal_distributions_is_zero_and_disjoint_is_one() {
|
||||
let mut p = BTreeMap::new();
|
||||
p.insert('a', 0.5);
|
||||
p.insert('b', 0.5);
|
||||
let same = p.clone();
|
||||
assert!(jsd(&p, &same).abs() < 1e-12);
|
||||
|
||||
let mut q = BTreeMap::new();
|
||||
q.insert('c', 1.0);
|
||||
assert!((jsd(&p, &q) - 1.0).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jsd_is_symmetric_and_finite_with_zero_support() {
|
||||
let mut p = BTreeMap::new();
|
||||
p.insert('a', 0.75);
|
||||
p.insert('b', 0.25);
|
||||
let mut q = BTreeMap::new();
|
||||
q.insert('b', 1.0);
|
||||
let forward = jsd(&p, &q);
|
||||
let backward = jsd(&q, &p);
|
||||
assert_eq!(forward.to_bits(), backward.to_bits());
|
||||
assert!(forward.is_finite());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base_domain_heuristic_matches_the_spec() {
|
||||
assert_eq!(
|
||||
split_base_domain("a.b.example.co.uk"),
|
||||
Some(("example.co.uk".to_string(), "a.b".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
split_base_domain("x.example.com"),
|
||||
Some(("example.com".to_string(), "x".to_string()))
|
||||
);
|
||||
assert_eq!(split_base_domain("example.com"), None);
|
||||
assert_eq!(classify_query("foo.local", 53), None);
|
||||
assert_eq!(classify_query("1.0.0.10.in-addr.arpa", 53), None);
|
||||
assert_eq!(classify_query("x.example.com", 5353), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthetic_base32_tunnel_is_detected_on_50_seeds() {
|
||||
for seed in 1..=50u64 {
|
||||
let mut rng = SplitMix64::new(seed * 1000 + 100);
|
||||
let mut queries = Vec::new();
|
||||
for i in 0..100 {
|
||||
let labels = 2 + (i % 2);
|
||||
let mut sub = Vec::new();
|
||||
for _ in 0..labels {
|
||||
sub.push(base32_label(&mut rng));
|
||||
}
|
||||
queries.push(DnsQuery {
|
||||
ts: 1_800_000_000.0 + (i as f64) * 30.0,
|
||||
orig_h: "10.66.66.66".to_string(),
|
||||
resp_p: 53,
|
||||
query: format!("{}.t.example-tunnel.net", sub.join(".")),
|
||||
qtype_name: Some("TXT".to_string()),
|
||||
rcode_name: Some("NOERROR".to_string()),
|
||||
});
|
||||
}
|
||||
let scores = score(&ProtocolInput { queries }).unwrap();
|
||||
let best = scores
|
||||
.iter()
|
||||
.max_by(|a, b| a.score.total_cmp(&b.score))
|
||||
.unwrap();
|
||||
assert!(best.score >= 0.40, "seed {seed}: score {}", best.score);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthetic_hex_tunnel_is_detected_on_50_seeds() {
|
||||
for seed in 1..=50u64 {
|
||||
let mut rng = SplitMix64::new(seed * 1000 + 101);
|
||||
let mut queries = Vec::new();
|
||||
for i in 0..100 {
|
||||
let len = 60 + rng.below(4) as usize;
|
||||
let label: String = (0..len).map(|_| hex_char(&mut rng)).collect();
|
||||
let len2 = 60 + rng.below(4) as usize;
|
||||
let label2: String = (0..len2).map(|_| hex_char(&mut rng)).collect();
|
||||
queries.push(DnsQuery {
|
||||
ts: 1_800_000_000.0 + (i as f64) * 30.0,
|
||||
orig_h: "10.66.66.67".to_string(),
|
||||
resp_p: 53,
|
||||
query: format!("{label}.{label2}.example.com"),
|
||||
qtype_name: Some("TXT".to_string()),
|
||||
rcode_name: Some("NOERROR".to_string()),
|
||||
});
|
||||
}
|
||||
let scores = score(&ProtocolInput { queries }).unwrap();
|
||||
let best = scores
|
||||
.iter()
|
||||
.max_by(|a, b| a.score.total_cmp(&b.score))
|
||||
.unwrap();
|
||||
assert!(best.score >= 0.40, "seed {seed}: score {}", best.score);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthetic_cdn_cloudfront_is_zero_and_amazonaws_stays_below_alert() {
|
||||
let mut rng = SplitMix64::new(42);
|
||||
let mut cloudfront = Vec::new();
|
||||
for i in 0..45 {
|
||||
let mut name = String::from("d");
|
||||
for _ in 0..13 {
|
||||
let alphabet = b"abcdefghijklmnopqrstuvwxyz0123456789";
|
||||
name.push(alphabet[rng.below(alphabet.len() as u64) as usize] as char);
|
||||
}
|
||||
cloudfront.push(DnsQuery {
|
||||
ts: 1_800_000_000.0 + (i as f64) * 60.0,
|
||||
orig_h: "10.0.2.15".to_string(),
|
||||
resp_p: 53,
|
||||
query: format!("{name}.cloudfront.net"),
|
||||
qtype_name: Some("A".to_string()),
|
||||
rcode_name: Some("NOERROR".to_string()),
|
||||
});
|
||||
}
|
||||
let scores = score(&ProtocolInput {
|
||||
queries: cloudfront,
|
||||
})
|
||||
.unwrap();
|
||||
let best = scores
|
||||
.iter()
|
||||
.max_by(|a, b| a.score.total_cmp(&b.score))
|
||||
.unwrap();
|
||||
assert_eq!(best.score, 0.0);
|
||||
assert!(best.information_bits < ZERO_BITS);
|
||||
|
||||
let mut amazon = Vec::new();
|
||||
for i in 0..70 {
|
||||
let a = rng.below(256);
|
||||
let b = rng.below(256);
|
||||
let c = rng.below(256);
|
||||
let d = rng.below(256);
|
||||
amazon.push(DnsQuery {
|
||||
ts: 1_800_000_000.0 + (i as f64) * 40.0,
|
||||
orig_h: "10.0.2.15".to_string(),
|
||||
resp_p: 53,
|
||||
query: format!("ec2-{a}-{b}-{c}-{d}.compute-1.amazonaws.com"),
|
||||
qtype_name: Some("A".to_string()),
|
||||
rcode_name: Some("NOERROR".to_string()),
|
||||
});
|
||||
}
|
||||
let scores = score(&ProtocolInput { queries: amazon }).unwrap();
|
||||
let best = scores
|
||||
.iter()
|
||||
.max_by(|a, b| a.score.total_cmp(&b.score))
|
||||
.unwrap();
|
||||
assert!(
|
||||
best.score < 0.40,
|
||||
"amazonaws score {} ({} bits)",
|
||||
best.score,
|
||||
best.information_bits
|
||||
);
|
||||
assert!(best.status == "ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evidence_is_byte_for_byte_deterministic() {
|
||||
let mut rng = SplitMix64::new(7);
|
||||
let mut queries = Vec::new();
|
||||
for i in 0..200 {
|
||||
let name = base32_label(&mut rng);
|
||||
let host = if i % 3 == 0 { "10.0.0.1" } else { "10.0.0.2" };
|
||||
let base = if i % 5 == 0 {
|
||||
"example.com"
|
||||
} else {
|
||||
"example.net"
|
||||
};
|
||||
queries.push(DnsQuery {
|
||||
ts: 1_800_000_000.0 + (i as f64) * 17.0,
|
||||
orig_h: host.to_string(),
|
||||
resp_p: 53,
|
||||
query: format!("{name}.{base}"),
|
||||
qtype_name: Some("A".to_string()),
|
||||
rcode_name: Some(if i % 11 == 0 { "NXDOMAIN" } else { "NOERROR" }.to_string()),
|
||||
});
|
||||
}
|
||||
let first = serde_json::to_vec(
|
||||
&score(&ProtocolInput {
|
||||
queries: queries.clone(),
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let second = serde_json::to_vec(&score(&ProtocolInput { queries }).unwrap()).unwrap();
|
||||
assert_eq!(first, second);
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,275 @@
|
||||
use crate::error::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub const WEIGHT_P1: f64 = 0.75;
|
||||
pub const WEIGHT_P2: f64 = 0.75;
|
||||
pub const WEIGHT_P3: f64 = 0.75;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct FusionWeights {
|
||||
pub p1: f64,
|
||||
pub p2: f64,
|
||||
pub p3: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct FusionTerm {
|
||||
pub pillar: u8,
|
||||
pub alert_id: Option<String>,
|
||||
pub m: f64,
|
||||
pub w: f64,
|
||||
pub weighted: f64,
|
||||
pub complement: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct FusionInput {
|
||||
pub beaconing: f64,
|
||||
pub sessions: f64,
|
||||
pub protocol: f64,
|
||||
pub m1: f64,
|
||||
pub m2: f64,
|
||||
pub m3: f64,
|
||||
#[serde(default)]
|
||||
pub alert_id_p1: Option<String>,
|
||||
#[serde(default)]
|
||||
pub alert_id_p2: Option<String>,
|
||||
#[serde(default)]
|
||||
pub alert_id_p3: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct FusionScore {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct HostScore {
|
||||
pub orig_h: String,
|
||||
pub score: f64,
|
||||
pub inputs: usize,
|
||||
pub terms: Vec<FusionTerm>,
|
||||
pub product: f64,
|
||||
pub method: &'static str,
|
||||
pub status: &'static str,
|
||||
pub weights: FusionWeights,
|
||||
pub order: [u8; 3],
|
||||
pub formula_version: &'static str,
|
||||
}
|
||||
|
||||
pub fn score(input: &FusionInput) -> Result<FusionScore> {
|
||||
Ok(FusionScore {
|
||||
score: 0.0,
|
||||
inputs: [input.beaconing, input.sessions, input.protocol].len(),
|
||||
method: "deterministic-fusion-stub",
|
||||
status: "skeleton",
|
||||
pub fn score(orig_h: &str, input: &FusionInput) -> Result<HostScore> {
|
||||
let t1_weighted = WEIGHT_P1 * input.m1;
|
||||
let t1_comp = 1.0 - t1_weighted;
|
||||
let term1 = FusionTerm {
|
||||
pillar: 1,
|
||||
alert_id: input.alert_id_p1.clone(),
|
||||
m: input.m1,
|
||||
w: WEIGHT_P1,
|
||||
weighted: t1_weighted,
|
||||
complement: t1_comp,
|
||||
};
|
||||
|
||||
let t2_weighted = WEIGHT_P2 * input.m2;
|
||||
let t2_comp = 1.0 - t2_weighted;
|
||||
let term2 = FusionTerm {
|
||||
pillar: 2,
|
||||
alert_id: input.alert_id_p2.clone(),
|
||||
m: input.m2,
|
||||
w: WEIGHT_P2,
|
||||
weighted: t2_weighted,
|
||||
complement: t2_comp,
|
||||
};
|
||||
|
||||
let t3_weighted = WEIGHT_P3 * input.m3;
|
||||
let t3_comp = 1.0 - t3_weighted;
|
||||
let term3 = FusionTerm {
|
||||
pillar: 3,
|
||||
alert_id: input.alert_id_p3.clone(),
|
||||
m: input.m3,
|
||||
w: WEIGHT_P3,
|
||||
weighted: t3_weighted,
|
||||
complement: t3_comp,
|
||||
};
|
||||
|
||||
// Product computed in fixed order 1, 2, 3
|
||||
let product = t1_comp * t2_comp * t3_comp;
|
||||
let s = (1.0 - product).clamp(0.0, 1.0);
|
||||
|
||||
Ok(HostScore {
|
||||
orig_h: orig_h.to_string(),
|
||||
score: s,
|
||||
terms: vec![term1, term2, term3],
|
||||
product,
|
||||
method: "weighted-noisy-or",
|
||||
weights: FusionWeights {
|
||||
p1: WEIGHT_P1,
|
||||
p2: WEIGHT_P2,
|
||||
p3: WEIGHT_P3,
|
||||
},
|
||||
order: [1, 2, 3],
|
||||
formula_version: "1.0",
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_1_noisy_or_properties() {
|
||||
// Zero input -> score 0
|
||||
let zero = score(
|
||||
"10.0.0.1",
|
||||
&FusionInput {
|
||||
m1: 0.0,
|
||||
m2: 0.0,
|
||||
m3: 0.0,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(zero.score, 0.0);
|
||||
|
||||
// Exact ceilings
|
||||
let one_pillar = score(
|
||||
"10.0.0.1",
|
||||
&FusionInput {
|
||||
m1: 1.0,
|
||||
m2: 0.0,
|
||||
m3: 0.0,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(one_pillar.score, 0.75);
|
||||
|
||||
let two_pillars = score(
|
||||
"10.0.0.1",
|
||||
&FusionInput {
|
||||
m1: 1.0,
|
||||
m2: 1.0,
|
||||
m3: 0.0,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(two_pillars.score, 0.9375);
|
||||
|
||||
let three_pillars = score(
|
||||
"10.0.0.1",
|
||||
&FusionInput {
|
||||
m1: 1.0,
|
||||
m2: 1.0,
|
||||
m3: 1.0,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(three_pillars.score, 0.984375);
|
||||
|
||||
// Monotonicity over 11x11x11 grid
|
||||
for i in 0..=10 {
|
||||
let m1 = i as f64 / 10.0;
|
||||
for j in 0..=10 {
|
||||
let m2 = j as f64 / 10.0;
|
||||
for k in 0..=10 {
|
||||
let m3 = k as f64 / 10.0;
|
||||
let base = score(
|
||||
"10.0.0.1",
|
||||
&FusionInput {
|
||||
m1,
|
||||
m2,
|
||||
m3,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.score;
|
||||
|
||||
if i < 10 {
|
||||
let step = score(
|
||||
"10.0.0.1",
|
||||
&FusionInput {
|
||||
m1: m1 + 0.1,
|
||||
m2,
|
||||
m3,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.score;
|
||||
assert!(step >= base);
|
||||
}
|
||||
if j < 10 {
|
||||
let step = score(
|
||||
"10.0.0.1",
|
||||
&FusionInput {
|
||||
m1,
|
||||
m2: m2 + 0.1,
|
||||
m3,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.score;
|
||||
assert!(step >= base);
|
||||
}
|
||||
if k < 10 {
|
||||
let step = score(
|
||||
"10.0.0.1",
|
||||
&FusionInput {
|
||||
m1,
|
||||
m2,
|
||||
m3: m3 + 0.1,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.score;
|
||||
assert!(step >= base);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_2_validation_table_vectors() {
|
||||
let cases = [
|
||||
("192.168.1.132", 1.000, 0.971, 0.0, 0.932), // Philips HUE benign
|
||||
("192.168.100.113", 0.829, 0.0, 0.0, 0.622), // Hakai C&C
|
||||
("192.168.1.195", 0.0, 0.821, 0.0, 0.616), // Mirai C&C
|
||||
("192.168.100.103", 0.0, 0.790, 0.0, 0.593), // Torii C&C
|
||||
("10.0.2.15", 0.0, 0.582, 0.0, 0.436), // CTU-Normal-26
|
||||
("192.168.1.114", 0.471, 0.0, 0.0, 0.353), // CTU-Normal-18
|
||||
("192.168.1.197", 0.0, 0.0, 0.0, 0.0), // 42-1 Trojan
|
||||
("10.0.2.15-20", 0.0, 0.0, 0.0, 0.0), // CTU-Normal-20
|
||||
];
|
||||
|
||||
for (host, m1, m2, m3, expected_s) in cases {
|
||||
let res = score(
|
||||
host,
|
||||
&FusionInput {
|
||||
m1,
|
||||
m2,
|
||||
m3,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
(res.score - expected_s).abs() < 1e-3,
|
||||
"host {host} score {} != expected {}",
|
||||
res.score,
|
||||
expected_s
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_3_determinism() {
|
||||
let input = FusionInput {
|
||||
m1: 0.829,
|
||||
m2: 0.790,
|
||||
m3: 0.450,
|
||||
alert_id_p1: Some("p1_id".to_string()),
|
||||
alert_id_p2: Some("p2_id".to_string()),
|
||||
alert_id_p3: Some("p3_id".to_string()),
|
||||
};
|
||||
let s1 = score("10.0.0.1", &input).unwrap();
|
||||
let s2 = score("10.0.0.1", &input).unwrap();
|
||||
let j1 = serde_json::to_vec(&s1).unwrap();
|
||||
let j2 = serde_json::to_vec(&s2).unwrap();
|
||||
assert_eq!(j1, j2);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
use crate::error::{AcauaError, Result};
|
||||
use chrono::NaiveDate;
|
||||
use serde::Serialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
@@ -27,12 +27,14 @@ impl DatasetSummary {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub struct IngestionStats {
|
||||
pub files_seen: usize,
|
||||
pub records_ingested: u64,
|
||||
pub lines_skipped: u64,
|
||||
pub partitions_affected: usize,
|
||||
pub files_failed: usize,
|
||||
pub failed_errors: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -40,9 +42,35 @@ pub struct TupleTimestamps {
|
||||
pub orig_h: String,
|
||||
pub resp_h: String,
|
||||
pub resp_p: u16,
|
||||
pub proto: String,
|
||||
pub timestamps: Vec<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChainedSession {
|
||||
pub orig_h: String,
|
||||
pub orig_p: u16,
|
||||
pub resp_h: String,
|
||||
pub resp_p: u16,
|
||||
pub proto: String,
|
||||
pub start_ts: f64,
|
||||
pub end_ts: f64,
|
||||
pub duration: f64,
|
||||
pub parts: usize,
|
||||
pub orig_bytes: u64,
|
||||
pub resp_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TupleSessions {
|
||||
pub orig_h: String,
|
||||
pub resp_h: String,
|
||||
pub resp_p: u16,
|
||||
pub proto: String,
|
||||
pub sessions: Vec<ChainedSession>,
|
||||
pub failed_attempts: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Storage {
|
||||
root: PathBuf,
|
||||
@@ -59,6 +87,12 @@ pub struct ConnRecord {
|
||||
pub proto: String,
|
||||
pub service: Option<String>,
|
||||
pub duration: Option<f64>,
|
||||
pub orig_bytes: Option<u64>,
|
||||
pub resp_bytes: Option<u64>,
|
||||
pub conn_state: Option<String>,
|
||||
pub history: Option<String>,
|
||||
pub orig_pkts: Option<u64>,
|
||||
pub resp_pkts: Option<u64>,
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
@@ -133,7 +167,7 @@ impl Storage {
|
||||
|
||||
let query = format!(
|
||||
"SELECT count(*)::BIGINT, count(DISTINCT date)::BIGINT
|
||||
FROM read_parquet('{parquet_glob}', hive_partitioning = true);"
|
||||
FROM read_parquet('{parquet_glob}', hive_partitioning = true, union_by_name = true);"
|
||||
);
|
||||
|
||||
let row = conn.query_row(&query, [], |r| {
|
||||
@@ -193,6 +227,26 @@ impl Storage {
|
||||
}
|
||||
}
|
||||
|
||||
// Also prune raw/dns if exists
|
||||
let dns_dir = self.root.join("raw").join("dns");
|
||||
if let Ok(dns_entries) = fs::read_dir(&dns_dir) {
|
||||
for entry in dns_entries.flatten() {
|
||||
let path = entry.path();
|
||||
if !path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
|
||||
if let Some(date_str) = name.strip_prefix("date=") {
|
||||
if let Ok(d) = NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||||
if d <= cutoff && fs::remove_dir_all(&path).is_ok() {
|
||||
pruned += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(pruned)
|
||||
}
|
||||
|
||||
@@ -203,6 +257,7 @@ impl Storage {
|
||||
|
||||
let mut separator = '\t';
|
||||
let mut unset_field = String::from("-");
|
||||
let mut empty_field = String::from("(empty)");
|
||||
let mut fields: Vec<String> = Vec::new();
|
||||
|
||||
let mut idx_ts = None;
|
||||
@@ -214,6 +269,12 @@ impl Storage {
|
||||
let mut idx_proto = None;
|
||||
let mut idx_service = None;
|
||||
let mut idx_duration = None;
|
||||
let mut idx_orig_bytes = None;
|
||||
let mut idx_resp_bytes = None;
|
||||
let mut idx_conn_state = None;
|
||||
let mut idx_history = None;
|
||||
let mut idx_orig_pkts = None;
|
||||
let mut idx_resp_pkts = None;
|
||||
|
||||
let text = String::from_utf8_lossy(content);
|
||||
|
||||
@@ -235,6 +296,8 @@ impl Storage {
|
||||
}
|
||||
} else if let Some(rest) = line.strip_prefix("#unset_field") {
|
||||
unset_field = rest.trim().to_string();
|
||||
} else if let Some(rest) = line.strip_prefix("#empty_field") {
|
||||
empty_field = rest.trim().to_string();
|
||||
} else if let Some(rest) = line.strip_prefix("#fields") {
|
||||
let rest = rest.trim();
|
||||
fields = rest
|
||||
@@ -251,11 +314,16 @@ impl Storage {
|
||||
idx_proto = fields.iter().position(|s| s == "proto");
|
||||
idx_service = fields.iter().position(|s| s == "service");
|
||||
idx_duration = fields.iter().position(|s| s == "duration");
|
||||
idx_orig_bytes = fields.iter().position(|s| s == "orig_bytes");
|
||||
idx_resp_bytes = fields.iter().position(|s| s == "resp_bytes");
|
||||
idx_conn_state = fields.iter().position(|s| s == "conn_state");
|
||||
idx_history = fields.iter().position(|s| s == "history");
|
||||
idx_orig_pkts = fields.iter().position(|s| s == "orig_pkts");
|
||||
idx_resp_pkts = fields.iter().position(|s| s == "resp_pkts");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Must have fields mapped
|
||||
let (i_ts, i_uid, i_oh, i_op, i_rh, i_rp, i_pr) = match (
|
||||
idx_ts, idx_uid, idx_orig_h, idx_orig_p, idx_resp_h, idx_resp_p, idx_proto,
|
||||
) {
|
||||
@@ -298,25 +366,29 @@ impl Storage {
|
||||
}
|
||||
};
|
||||
|
||||
let service = idx_service.and_then(|idx| {
|
||||
let val = parts.get(idx)?;
|
||||
if *val == unset_field.as_str() || *val == "-" {
|
||||
let get_val = |idx: Option<usize>| -> Option<&str> {
|
||||
let i = idx?;
|
||||
let val = parts.get(i)?;
|
||||
if *val == unset_field.as_str()
|
||||
|| *val == "-"
|
||||
|| *val == empty_field.as_str()
|
||||
|| *val == "(empty)"
|
||||
{
|
||||
None
|
||||
} else {
|
||||
Some(val.to_string())
|
||||
Some(*val)
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
let duration = idx_duration.and_then(|idx| {
|
||||
let val = parts.get(idx)?;
|
||||
if *val == unset_field.as_str() || *val == "-" {
|
||||
None
|
||||
} else {
|
||||
val.parse::<f64>().ok()
|
||||
}
|
||||
});
|
||||
let service = get_val(idx_service).map(|s| s.to_string());
|
||||
let duration = get_val(idx_duration).and_then(|s| s.parse::<f64>().ok());
|
||||
let orig_bytes = get_val(idx_orig_bytes).and_then(|s| s.parse::<u64>().ok());
|
||||
let resp_bytes = get_val(idx_resp_bytes).and_then(|s| s.parse::<u64>().ok());
|
||||
let conn_state = get_val(idx_conn_state).map(|s| s.to_string());
|
||||
let history = get_val(idx_history).map(|s| s.to_string());
|
||||
let orig_pkts = get_val(idx_orig_pkts).and_then(|s| s.parse::<u64>().ok());
|
||||
let resp_pkts = get_val(idx_resp_pkts).and_then(|s| s.parse::<u64>().ok());
|
||||
|
||||
// Track day partition (UTC)
|
||||
let date_ts = chrono::DateTime::from_timestamp(ts.floor() as i64, 0);
|
||||
if let Some(dt) = date_ts {
|
||||
dates.insert(dt.format("%Y-%m-%d").to_string());
|
||||
@@ -332,13 +404,19 @@ impl Storage {
|
||||
proto: parts[i_pr].to_string(),
|
||||
service,
|
||||
duration,
|
||||
orig_bytes,
|
||||
resp_bytes,
|
||||
conn_state,
|
||||
history,
|
||||
orig_pkts,
|
||||
resp_pkts,
|
||||
});
|
||||
}
|
||||
|
||||
(records, skipped, dates)
|
||||
}
|
||||
|
||||
pub fn ingest_file(&self, path: impl AsRef<Path>) -> Result<(u64, u64, usize)> {
|
||||
pub fn ingest_file(&self, path: impl AsRef<Path>) -> Result<(u64, u64, HashSet<String>)> {
|
||||
let path = path.as_ref();
|
||||
let content = fs::read(path)?;
|
||||
let mut hasher = Sha256::new();
|
||||
@@ -352,7 +430,7 @@ impl Storage {
|
||||
|
||||
let (records, skipped, dates) = Self::parse_conn_log(&content);
|
||||
if records.is_empty() {
|
||||
return Ok((0, skipped, 0));
|
||||
return Ok((0, skipped, dates));
|
||||
}
|
||||
|
||||
let conn = self.open_connection()?;
|
||||
@@ -366,7 +444,13 @@ impl Storage {
|
||||
resp_p USMALLINT,
|
||||
proto VARCHAR,
|
||||
service VARCHAR,
|
||||
duration DOUBLE
|
||||
duration DOUBLE,
|
||||
orig_bytes UBIGINT,
|
||||
resp_bytes UBIGINT,
|
||||
conn_state VARCHAR,
|
||||
history VARCHAR,
|
||||
orig_pkts UBIGINT,
|
||||
resp_pkts UBIGINT
|
||||
);",
|
||||
)?;
|
||||
|
||||
@@ -374,8 +458,21 @@ impl Storage {
|
||||
let mut appender = conn.appender("t")?;
|
||||
for r in &records {
|
||||
appender.append_row(duckdb::params![
|
||||
r.ts, r.uid, r.orig_h, r.orig_p, r.resp_h, r.resp_p, r.proto, r.service,
|
||||
r.duration
|
||||
r.ts,
|
||||
r.uid,
|
||||
r.orig_h,
|
||||
r.orig_p,
|
||||
r.resp_h,
|
||||
r.resp_p,
|
||||
r.proto,
|
||||
r.service,
|
||||
r.duration,
|
||||
r.orig_bytes,
|
||||
r.resp_bytes,
|
||||
r.conn_state,
|
||||
r.history,
|
||||
r.orig_pkts,
|
||||
r.resp_pkts
|
||||
])?;
|
||||
}
|
||||
}
|
||||
@@ -401,7 +498,7 @@ impl Storage {
|
||||
conn.execute_batch(©_sql)?;
|
||||
conn.execute_batch("DROP TABLE t;")?;
|
||||
|
||||
Ok((records.len() as u64, skipped, dates.len()))
|
||||
Ok((records.len() as u64, skipped, dates))
|
||||
}
|
||||
|
||||
pub fn ingest_dir(&self, directory: impl AsRef<Path>) -> Result<IngestionStats> {
|
||||
@@ -421,7 +518,6 @@ impl Storage {
|
||||
let entries = fs::read_dir(&canonical_dir)?;
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
// Path confinement: do not follow symlink to outside directory
|
||||
if let Ok(canon) = fs::canonicalize(&path) {
|
||||
if !canon.starts_with(&canonical_dir) {
|
||||
continue;
|
||||
@@ -435,27 +531,26 @@ impl Storage {
|
||||
None => continue,
|
||||
};
|
||||
|
||||
// Skip .gz archives with note
|
||||
if file_name.ends_with(".gz") {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Only conn.log and conn.*.log directly in directory
|
||||
let is_conn = file_name == "conn.log"
|
||||
|| (file_name.starts_with("conn.") && file_name.ends_with(".log"));
|
||||
|
||||
if is_conn && path.is_file() {
|
||||
stats.files_seen += 1;
|
||||
match self.ingest_file(&path) {
|
||||
Ok((recs, sk, affected)) => {
|
||||
Ok((recs, sk, dates)) => {
|
||||
stats.records_ingested += recs;
|
||||
stats.lines_skipped += sk;
|
||||
for _ in 0..affected {
|
||||
all_dates.insert(file_name.to_string());
|
||||
}
|
||||
all_dates.extend(dates);
|
||||
}
|
||||
Err(_) => {
|
||||
stats.lines_skipped += 1;
|
||||
Err(e) => {
|
||||
stats.files_failed += 1;
|
||||
if stats.failed_errors.len() < 10 {
|
||||
stats.failed_errors.push(e.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -465,7 +560,7 @@ impl Storage {
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
pub fn query_7day_tuples(&self) -> Result<Vec<TupleTimestamps>> {
|
||||
pub fn query_7day_sessions(&self) -> Result<Vec<TupleSessions>> {
|
||||
if !self.has_parquet_files() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
@@ -483,7 +578,7 @@ impl Storage {
|
||||
// Find max date present in data
|
||||
let max_date_query = format!(
|
||||
"SELECT max(date)::VARCHAR
|
||||
FROM read_parquet('{parquet_glob}', hive_partitioning = true);"
|
||||
FROM read_parquet('{parquet_glob}', hive_partitioning = true, union_by_name = true);"
|
||||
);
|
||||
|
||||
let max_date: Option<String> = conn
|
||||
@@ -495,47 +590,193 @@ impl Storage {
|
||||
_ => return Ok(Vec::new()),
|
||||
};
|
||||
|
||||
// Query all conn rows in 7-day window ordered by orig_h, orig_p, resp_h, resp_p, ts
|
||||
let query = format!(
|
||||
"WITH daily AS (
|
||||
SELECT date, orig_h, resp_h, resp_p, list(ts ORDER BY ts) AS tss
|
||||
FROM read_parquet('{parquet_glob}', hive_partitioning = true)
|
||||
WHERE date > CAST('{max_date_str}' AS DATE) - INTERVAL 7 DAY
|
||||
GROUP BY ALL
|
||||
)
|
||||
SELECT orig_h, resp_h, resp_p, list_sort(flatten(list(tss))) AS timestamps
|
||||
FROM daily
|
||||
GROUP BY ALL
|
||||
ORDER BY orig_h, resp_h, resp_p;"
|
||||
"SELECT ts, orig_h, orig_p, resp_h, resp_p, proto, duration, orig_bytes, resp_bytes, conn_state, history
|
||||
FROM read_parquet('{parquet_glob}', hive_partitioning = true, union_by_name = true)
|
||||
WHERE date > CAST('{max_date_str}' AS DATE) - INTERVAL 7 DAY
|
||||
ORDER BY orig_h, orig_p, resp_h, resp_p, ts;"
|
||||
);
|
||||
|
||||
struct RawRow {
|
||||
ts: f64,
|
||||
orig_h: String,
|
||||
orig_p: u16,
|
||||
resp_h: String,
|
||||
resp_p: u16,
|
||||
proto: String,
|
||||
duration: Option<f64>,
|
||||
orig_bytes: Option<u64>,
|
||||
resp_bytes: Option<u64>,
|
||||
conn_state: Option<String>,
|
||||
history: Option<String>,
|
||||
}
|
||||
|
||||
let mut stmt = conn.prepare(&query)?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
let orig_h: String = row.get(0)?;
|
||||
let resp_h: String = row.get(1)?;
|
||||
let resp_p: u16 = row.get(2)?;
|
||||
let timestamps_val: duckdb::types::Value = row.get(3)?;
|
||||
let mut timestamps = Vec::new();
|
||||
if let duckdb::types::Value::List(items) = timestamps_val {
|
||||
for item in items {
|
||||
if let duckdb::types::Value::Double(t) = item {
|
||||
timestamps.push(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(TupleTimestamps {
|
||||
orig_h,
|
||||
resp_h,
|
||||
resp_p,
|
||||
timestamps,
|
||||
Ok(RawRow {
|
||||
ts: row.get(0)?,
|
||||
orig_h: row.get(1)?,
|
||||
orig_p: row.get(2)?,
|
||||
resp_h: row.get(3)?,
|
||||
resp_p: row.get(4)?,
|
||||
proto: row.get(5)?,
|
||||
duration: row.get(6)?,
|
||||
orig_bytes: row.get(7)?,
|
||||
resp_bytes: row.get(8)?,
|
||||
conn_state: row.get(9)?,
|
||||
history: row.get(10)?,
|
||||
})
|
||||
})?;
|
||||
|
||||
let mut result = Vec::new();
|
||||
let mut raw_rows = Vec::new();
|
||||
let mut failed_map: HashMap<(String, String, u16), usize> = HashMap::new();
|
||||
|
||||
for r in rows {
|
||||
result.push(r?);
|
||||
let row = r?;
|
||||
if row.proto == "tcp" {
|
||||
if let Some(cs) = &row.conn_state {
|
||||
if cs == "S0" || cs == "REJ" || cs == "RSTOS0" || cs == "SH" {
|
||||
*failed_map
|
||||
.entry((row.orig_h.clone(), row.resp_h.clone(), row.resp_p))
|
||||
.or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
raw_rows.push(row);
|
||||
}
|
||||
|
||||
// Group rows by 4-tuple (orig_h, orig_p, resp_h, resp_p, proto)
|
||||
// Since rows are already ordered by (orig_h, orig_p, resp_h, resp_p, ts),
|
||||
// we can chain sequentially.
|
||||
let mut all_sessions: Vec<ChainedSession> = Vec::new();
|
||||
let mut current_chain: Option<ChainedSession> = None;
|
||||
|
||||
for r in raw_rows {
|
||||
let row_end = r.ts + r.duration.unwrap_or(0.0);
|
||||
if r.proto == "tcp" {
|
||||
let matches_current_4tuple = match ¤t_chain {
|
||||
Some(curr) => {
|
||||
curr.orig_h == r.orig_h
|
||||
&& curr.orig_p == r.orig_p
|
||||
&& curr.resp_h == r.resp_h
|
||||
&& curr.resp_p == r.resp_p
|
||||
&& curr.proto == r.proto
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
|
||||
let is_continuation = if matches_current_4tuple {
|
||||
let curr = current_chain.as_ref().unwrap();
|
||||
match &r.history {
|
||||
Some(h) => !h.contains('S') && (r.ts - curr.end_ts <= 3600.0),
|
||||
None => false,
|
||||
}
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
if is_continuation {
|
||||
let curr = current_chain.as_mut().unwrap();
|
||||
if row_end > curr.end_ts {
|
||||
curr.end_ts = row_end;
|
||||
}
|
||||
curr.duration = curr.end_ts - curr.start_ts;
|
||||
curr.parts += 1;
|
||||
curr.orig_bytes += r.orig_bytes.unwrap_or(0);
|
||||
curr.resp_bytes += r.resp_bytes.unwrap_or(0);
|
||||
} else {
|
||||
if let Some(finished) = current_chain.take() {
|
||||
all_sessions.push(finished);
|
||||
}
|
||||
current_chain = Some(ChainedSession {
|
||||
orig_h: r.orig_h,
|
||||
orig_p: r.orig_p,
|
||||
resp_h: r.resp_h,
|
||||
resp_p: r.resp_p,
|
||||
proto: r.proto,
|
||||
start_ts: r.ts,
|
||||
end_ts: row_end,
|
||||
duration: row_end - r.ts,
|
||||
parts: 1,
|
||||
orig_bytes: r.orig_bytes.unwrap_or(0),
|
||||
resp_bytes: r.resp_bytes.unwrap_or(0),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
if let Some(finished) = current_chain.take() {
|
||||
all_sessions.push(finished);
|
||||
}
|
||||
all_sessions.push(ChainedSession {
|
||||
orig_h: r.orig_h,
|
||||
orig_p: r.orig_p,
|
||||
resp_h: r.resp_h,
|
||||
resp_p: r.resp_p,
|
||||
proto: r.proto,
|
||||
start_ts: r.ts,
|
||||
end_ts: row_end,
|
||||
duration: row_end - r.ts,
|
||||
parts: 1,
|
||||
orig_bytes: r.orig_bytes.unwrap_or(0),
|
||||
resp_bytes: r.resp_bytes.unwrap_or(0),
|
||||
});
|
||||
}
|
||||
}
|
||||
if let Some(finished) = current_chain {
|
||||
all_sessions.push(finished);
|
||||
}
|
||||
|
||||
// Group sessions by (orig_h, resp_h, resp_p, proto)
|
||||
let mut grouped: HashMap<(String, String, u16, String), Vec<ChainedSession>> =
|
||||
HashMap::new();
|
||||
for session in all_sessions {
|
||||
let key = (
|
||||
session.orig_h.clone(),
|
||||
session.resp_h.clone(),
|
||||
session.resp_p,
|
||||
session.proto.clone(),
|
||||
);
|
||||
grouped.entry(key).or_default().push(session);
|
||||
}
|
||||
|
||||
let mut result = Vec::new();
|
||||
for ((orig_h, resp_h, resp_p, proto), mut sessions) in grouped {
|
||||
sessions.sort_by(|a, b| a.start_ts.partial_cmp(&b.start_ts).unwrap());
|
||||
let failed_attempts = failed_map
|
||||
.get(&(orig_h.clone(), resp_h.clone(), resp_p))
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
result.push(TupleSessions {
|
||||
orig_h,
|
||||
resp_h,
|
||||
resp_p,
|
||||
proto,
|
||||
sessions,
|
||||
failed_attempts,
|
||||
});
|
||||
}
|
||||
|
||||
result.sort_by(|a, b| {
|
||||
(&a.orig_h, &a.resp_h, a.resp_p, &a.proto)
|
||||
.cmp(&(&b.orig_h, &b.resp_h, b.resp_p, &b.proto))
|
||||
});
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn query_7day_tuples(&self) -> Result<Vec<TupleTimestamps>> {
|
||||
let tuple_sessions = self.query_7day_sessions()?;
|
||||
Ok(tuple_sessions
|
||||
.into_iter()
|
||||
.map(|ts| TupleTimestamps {
|
||||
orig_h: ts.orig_h,
|
||||
resp_h: ts.resp_h,
|
||||
resp_p: ts.resp_p,
|
||||
proto: ts.proto,
|
||||
timestamps: ts.sessions.into_iter().map(|s| s.start_ts).collect(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -548,15 +789,15 @@ mod tests {
|
||||
#unset_field\t-\n\
|
||||
#path\tconn\n\
|
||||
#open\t2026-10-31-12-00-00\n\
|
||||
#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\tservice\tduration\tip_proto\n\
|
||||
#types\ttime\tstring\taddr\tport\taddr\tport\tenum\tstring\tinterval\tcount\n\
|
||||
1793461200.000000\tCuid1\t192.168.1.10\t45000\t10.0.0.1\t443\ttcp\tssl\t1.234\t6\n\
|
||||
1793461205.000000\tCuid2\t192.168.1.10\t45001\t10.0.0.1\t443\ttcp\t-\t-\t6\n\
|
||||
#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\tservice\tduration\torig_bytes\tresp_bytes\tconn_state\thistory\torig_pkts\tresp_pkts\n\
|
||||
#types\ttime\tstring\taddr\tport\taddr\tport\tenum\tstring\tinterval\tcount\tcount\tstring\tstring\tcount\tcount\n\
|
||||
1793461200.000000\tCuid1\t192.168.1.10\t45000\t10.0.0.1\t443\ttcp\tssl\t1.234\t100\t200\tSF\tShADadFf\t10\t12\n\
|
||||
1793461205.000000\tCuid2\t192.168.1.10\t45001\t10.0.0.1\t443\ttcp\t-\t-\t-\t-\tS0\tS\t1\t0\n\
|
||||
#close\t2026-10-31-12-01-00\n";
|
||||
|
||||
const CONN_LOG_PERMUTED_COLS: &str = "#separator \\x09\n\
|
||||
#fields\tid.resp_p\tid.orig_h\tservice\tuid\tduration\tproto\tts\tid.resp_h\tid.orig_p\n\
|
||||
443\t192.168.1.10\thttp\tCuid3\t0.5\ttcp\t1793461210.000000\t10.0.0.1\t45002\n";
|
||||
#fields\tid.resp_p\tid.orig_h\tservice\tuid\tduration\tproto\tts\tid.resp_h\tid.orig_p\tconn_state\thistory\n\
|
||||
443\t192.168.1.10\thttp\tCuid3\t0.5\ttcp\t1793461210.000000\t10.0.0.1\t45002\tSF\tShAD\n";
|
||||
|
||||
#[test]
|
||||
fn test_parser_extracts_correct_fields_and_unset_as_none() {
|
||||
@@ -573,10 +814,19 @@ mod tests {
|
||||
assert_eq!(records[0].proto, "tcp");
|
||||
assert_eq!(records[0].service.as_deref(), Some("ssl"));
|
||||
assert_eq!(records[0].duration, Some(1.234));
|
||||
assert_eq!(records[0].orig_bytes, Some(100));
|
||||
assert_eq!(records[0].resp_bytes, Some(200));
|
||||
assert_eq!(records[0].conn_state.as_deref(), Some("SF"));
|
||||
assert_eq!(records[0].history.as_deref(), Some("ShADadFf"));
|
||||
assert_eq!(records[0].orig_pkts, Some(10));
|
||||
assert_eq!(records[0].resp_pkts, Some(12));
|
||||
|
||||
assert_eq!(records[1].uid, "Cuid2");
|
||||
assert_eq!(records[1].service, None);
|
||||
assert_eq!(records[1].duration, None);
|
||||
assert_eq!(records[1].orig_bytes, None);
|
||||
assert_eq!(records[1].conn_state.as_deref(), Some("S0"));
|
||||
assert_eq!(records[1].history.as_deref(), Some("S"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -611,7 +861,6 @@ mod tests {
|
||||
assert_eq!(summary1.records, 2);
|
||||
assert_eq!(summary1.partitions, 1);
|
||||
|
||||
// Idempotence: re-ingesting the exact same file does not duplicate records
|
||||
let (ingested2, skipped2, _) = store.ingest_file(&log_file).unwrap();
|
||||
assert_eq!(ingested2, 2);
|
||||
assert_eq!(skipped2, 0);
|
||||
@@ -619,7 +868,6 @@ mod tests {
|
||||
let summary2 = store.summary().unwrap();
|
||||
assert_eq!(summary2.records, 2);
|
||||
|
||||
// Ingesting a different file for the same tuple/day adds to it
|
||||
let log_file_2 = temp_dir.path().join("conn.extra.log");
|
||||
fs::write(&log_file_2, CONN_LOG_PERMUTED_COLS).unwrap();
|
||||
store.ingest_file(&log_file_2).unwrap();
|
||||
@@ -632,6 +880,7 @@ mod tests {
|
||||
assert_eq!(tuples[0].orig_h, "192.168.1.10");
|
||||
assert_eq!(tuples[0].resp_h, "10.0.0.1");
|
||||
assert_eq!(tuples[0].resp_p, 443);
|
||||
assert_eq!(tuples[0].proto, "tcp");
|
||||
assert_eq!(tuples[0].timestamps.len(), 3);
|
||||
assert_eq!(tuples[0].timestamps[0], 1793461200.0);
|
||||
assert_eq!(tuples[0].timestamps[1], 1793461205.0);
|
||||
@@ -643,7 +892,6 @@ mod tests {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let store = Storage::open(temp_dir.path()).unwrap();
|
||||
|
||||
// 2026-10-31 02:00:00 UTC = 1793412000
|
||||
let log = "#separator \\x09\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\n1793412000.000000\tu1\t10.0.0.1\t1000\t10.0.0.2\t80\ttcp\n";
|
||||
let log_file = temp_dir.path().join("conn.log");
|
||||
fs::write(&log_file, log).unwrap();
|
||||
@@ -676,7 +924,6 @@ mod tests {
|
||||
let store = Storage::open(temp_dir.path()).unwrap();
|
||||
|
||||
let raw_conn = temp_dir.path().join("raw").join("conn");
|
||||
// Day 1 (10 days ago) and Day 10 (recent)
|
||||
let old_part = raw_conn.join("date=2026-10-01");
|
||||
let new_part = raw_conn.join("date=2026-10-11");
|
||||
fs::create_dir_all(&old_part).unwrap();
|
||||
@@ -687,4 +934,154 @@ mod tests {
|
||||
assert!(!old_part.exists());
|
||||
assert!(new_part.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_union_by_name_v11_and_v12_compatibility() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let store = Storage::open(temp_dir.path()).unwrap();
|
||||
let conn = store.open_connection().unwrap();
|
||||
|
||||
let part_dir = store
|
||||
.root()
|
||||
.join("raw")
|
||||
.join("conn")
|
||||
.join("date=2026-10-31");
|
||||
fs::create_dir_all(&part_dir).unwrap();
|
||||
let part_str = part_dir.to_string_lossy().replace('\'', "''");
|
||||
|
||||
// Write v1.1 format (no history, no orig_bytes etc.)
|
||||
conn.execute_batch(&format!(
|
||||
"CREATE TEMP TABLE v11 (ts DOUBLE, uid VARCHAR, orig_h VARCHAR, orig_p USMALLINT, resp_h VARCHAR, resp_p USMALLINT, proto VARCHAR, date DATE);
|
||||
INSERT INTO v11 VALUES (1793461200.0, 'u1', '10.0.0.1', 1000, '10.0.0.2', 80, 'tcp', '2026-10-31');
|
||||
COPY v11 TO '{part_str}/v11.parquet' (FORMAT parquet);
|
||||
DROP TABLE v11;"
|
||||
)).unwrap();
|
||||
|
||||
// Write v1.2 format (with history, orig_bytes)
|
||||
conn.execute_batch(&format!(
|
||||
"CREATE TEMP TABLE v12 (ts DOUBLE, uid VARCHAR, orig_h VARCHAR, orig_p USMALLINT, resp_h VARCHAR, resp_p USMALLINT, proto VARCHAR, history VARCHAR, orig_bytes UBIGINT, date DATE);
|
||||
INSERT INTO v12 VALUES (1793461210.0, 'u2', '10.0.0.1', 1001, '10.0.0.2', 80, 'tcp', 'ShAD', 500, '2026-10-31');
|
||||
COPY v12 TO '{part_str}/v12.parquet' (FORMAT parquet);
|
||||
DROP TABLE v12;"
|
||||
)).unwrap();
|
||||
|
||||
let parquet_glob = store
|
||||
.root()
|
||||
.join("raw")
|
||||
.join("conn")
|
||||
.join("*")
|
||||
.join("*.parquet")
|
||||
.to_string_lossy()
|
||||
.replace('\'', "''");
|
||||
|
||||
let read_sql = format!(
|
||||
"SELECT count(*)::BIGINT, count(history)::BIGINT
|
||||
FROM read_parquet('{parquet_glob}', hive_partitioning = true, union_by_name = true);"
|
||||
);
|
||||
let (total, with_history): (i64, i64) = conn
|
||||
.query_row(&read_sql, [], |r| Ok((r.get(0)?, r.get(1)?)))
|
||||
.unwrap();
|
||||
assert_eq!(total, 2);
|
||||
assert_eq!(with_history, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_zeek_split_real_fixture_chaining() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let store = Storage::open(temp_dir.path()).unwrap();
|
||||
|
||||
let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
.join("zeek_split")
|
||||
.join("conn.log");
|
||||
|
||||
store.ingest_file(&fixture_path).unwrap();
|
||||
|
||||
let sessions = store.query_7day_sessions().unwrap();
|
||||
assert_eq!(sessions.len(), 1);
|
||||
assert_eq!(sessions[0].orig_h, "10.0.0.5");
|
||||
assert_eq!(sessions[0].resp_h, "203.0.113.9");
|
||||
assert_eq!(sessions[0].resp_p, 443);
|
||||
assert_eq!(sessions[0].proto, "tcp");
|
||||
assert_eq!(sessions[0].sessions.len(), 1);
|
||||
|
||||
let s = &sessions[0].sessions[0];
|
||||
assert_eq!(s.parts, 2);
|
||||
assert!(
|
||||
(s.duration - 4201.05).abs() < 0.01,
|
||||
"duration: expected 4201.05, got {}",
|
||||
s.duration
|
||||
);
|
||||
assert_eq!(s.orig_bytes, 1200);
|
||||
assert_eq!(s.resp_bytes, 2400);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chaining_negative_cases() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let store = Storage::open(temp_dir.path()).unwrap();
|
||||
|
||||
// 1. Second line has 'S' in history -> 2 sessions
|
||||
let log_syn = "#separator \\x09\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\tduration\thistory\n\
|
||||
1793400000.0\tu1\t10.0.0.1\t40000\t10.0.0.2\t80\ttcp\t100.0\tShAD\n\
|
||||
1793400200.0\tu2\t10.0.0.1\t40000\t10.0.0.2\t80\ttcp\t100.0\tShAD\n";
|
||||
let f_syn = temp_dir.path().join("conn.syn.log");
|
||||
fs::write(&f_syn, log_syn).unwrap();
|
||||
store.ingest_file(&f_syn).unwrap();
|
||||
|
||||
let sess_syn = store.query_7day_sessions().unwrap();
|
||||
assert_eq!(
|
||||
sess_syn[0].sessions.len(),
|
||||
2,
|
||||
"history with 'S' must not chain"
|
||||
);
|
||||
|
||||
// 2. Gap of 3601s -> 2 sessions
|
||||
let temp_dir2 = tempfile::tempdir().unwrap();
|
||||
let store2 = Storage::open(temp_dir2.path()).unwrap();
|
||||
let log_gap = "#separator \\x09\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\tduration\thistory\n\
|
||||
1793400000.0\tu1\t10.0.0.1\t40000\t10.0.0.2\t80\ttcp\t100.0\tShAD\n\
|
||||
1793403702.0\tu2\t10.0.0.1\t40000\t10.0.0.2\t80\ttcp\t100.0\tAD\n";
|
||||
let f_gap = temp_dir2.path().join("conn.gap.log");
|
||||
fs::write(&f_gap, log_gap).unwrap();
|
||||
store2.ingest_file(&f_gap).unwrap();
|
||||
|
||||
let sess_gap = store2.query_7day_sessions().unwrap();
|
||||
assert_eq!(sess_gap[0].sessions.len(), 2, "gap > 3600s must not chain");
|
||||
|
||||
// 3. history is NULL -> does not chain
|
||||
let temp_dir3 = tempfile::tempdir().unwrap();
|
||||
let store3 = Storage::open(temp_dir3.path()).unwrap();
|
||||
let log_null = "#separator \\x09\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\tduration\thistory\n\
|
||||
1793400000.0\tu1\t10.0.0.1\t40000\t10.0.0.2\t80\ttcp\t100.0\tShAD\n\
|
||||
1793400200.0\tu2\t10.0.0.1\t40000\t10.0.0.2\t80\ttcp\t100.0\t-\n";
|
||||
let f_null = temp_dir3.path().join("conn.null.log");
|
||||
fs::write(&f_null, log_null).unwrap();
|
||||
store3.ingest_file(&f_null).unwrap();
|
||||
|
||||
let sess_null = store3.query_7day_sessions().unwrap();
|
||||
assert_eq!(
|
||||
sess_null[0].sessions.len(),
|
||||
2,
|
||||
"history NULL must not chain"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partitions_affected_and_files_failed() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let store = Storage::open(temp_dir.path()).unwrap();
|
||||
|
||||
// 2 lines across 2 days (2026-10-30 and 2026-10-31)
|
||||
let log = "#separator \\x09\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\n\
|
||||
1793325600.0\tu1\t10.0.0.1\t1000\t10.0.0.2\t80\ttcp\n\
|
||||
1793412000.0\tu2\t10.0.0.1\t1000\t10.0.0.2\t80\ttcp\n";
|
||||
let f = temp_dir.path().join("conn.log");
|
||||
fs::write(&f, log).unwrap();
|
||||
|
||||
let stats = store.ingest_dir(temp_dir.path()).unwrap();
|
||||
assert_eq!(stats.partitions_affected, 2);
|
||||
assert_eq!(stats.files_failed, 0);
|
||||
}
|
||||
}
|
||||
+279
-30
@@ -1,6 +1,9 @@
|
||||
use crate::error::{AcauaError, Result};
|
||||
use crate::fusion::Alert;
|
||||
use crate::pillar1::{self, BeaconingInput};
|
||||
use crate::pillar2::{self, SessionCandidate, SessionInput};
|
||||
use crate::pillar3::{self, ProtocolInput};
|
||||
use crate::pillar4::HostScore;
|
||||
use crate::storage::Storage;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -21,13 +24,76 @@ pub enum ZeekLogKind {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct P1ExcludedCounts {
|
||||
pub non_tcp_udp: usize,
|
||||
pub multicast_broadcast: usize,
|
||||
pub infra_port: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AnalysisReport {
|
||||
pub directory: String,
|
||||
pub logs_seen: usize,
|
||||
pub dns_logs_seen: usize,
|
||||
pub records: u64,
|
||||
pub alerts: Vec<Alert>,
|
||||
pub hosts: Vec<HostScore>,
|
||||
pub evidence_schema_version: u32,
|
||||
pub status: &'static str,
|
||||
pub p1_excluded: P1ExcludedCounts,
|
||||
}
|
||||
|
||||
pub fn classify_log(file_name: &str) -> ZeekLogKind {
|
||||
match file_name {
|
||||
"conn.log" => ZeekLogKind::Conn,
|
||||
"dns.log" => ZeekLogKind::Dns,
|
||||
"ssl.log" => ZeekLogKind::Ssl,
|
||||
"smb_mapping.log" => ZeekLogKind::SmbMapping,
|
||||
"dce_rpc.log" => ZeekLogKind::DceRpc,
|
||||
"rdp.log" => ZeekLogKind::Rdp,
|
||||
"ssh.log" => ZeekLogKind::Ssh,
|
||||
"kerberos.log" => ZeekLogKind::Kerberos,
|
||||
"ntlm.log" => ZeekLogKind::Ntlm,
|
||||
_ => ZeekLogKind::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_unicast(ip_str: &str) -> bool {
|
||||
if ip_str == "255.255.255.255" {
|
||||
return false;
|
||||
}
|
||||
if let Ok(ip) = ip_str.parse::<std::net::IpAddr>() {
|
||||
!ip.is_multicast()
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
fn is_infra_port(proto: &str, port: u16) -> bool {
|
||||
matches!(
|
||||
(proto, port),
|
||||
("udp", 53)
|
||||
| ("tcp", 53)
|
||||
| ("udp", 123)
|
||||
| ("udp", 67)
|
||||
| ("udp", 68)
|
||||
| ("udp", 546)
|
||||
| ("udp", 547)
|
||||
| ("udp", 137)
|
||||
| ("udp", 138)
|
||||
| ("udp", 1900)
|
||||
| ("udp", 5353)
|
||||
| ("udp", 5355)
|
||||
)
|
||||
}
|
||||
|
||||
struct AlertItem {
|
||||
pillar: u8,
|
||||
orig_h: String,
|
||||
dest: String,
|
||||
resp_p: u16,
|
||||
alert: Alert,
|
||||
}
|
||||
|
||||
pub fn analyze_dir(
|
||||
@@ -43,22 +109,41 @@ pub fn analyze_dir(
|
||||
}
|
||||
|
||||
let store = Storage::open(data_dir)?;
|
||||
let stats = store.ingest_dir(directory)?;
|
||||
let tuples = store.query_7day_tuples()?;
|
||||
let mut stats = store.ingest_dir(directory)?;
|
||||
let tuple_sessions = store.query_7day_sessions()?;
|
||||
|
||||
let mut alerts = Vec::new();
|
||||
let mut alert_items: Vec<AlertItem> = Vec::new();
|
||||
let mut p1_excluded = P1ExcludedCounts::default();
|
||||
|
||||
for tuple in tuples {
|
||||
if tuple.timestamps.len() < 20 {
|
||||
// Pilar 1 evaluation: low-and-slow C2 beaconing detection (formula v1.2)
|
||||
for tuple in &tuple_sessions {
|
||||
if tuple.proto != "tcp" && tuple.proto != "udp" {
|
||||
p1_excluded.non_tcp_udp += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if !is_unicast(&tuple.resp_h) {
|
||||
p1_excluded.multicast_broadcast += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_infra_port(&tuple.proto, tuple.resp_p) {
|
||||
p1_excluded.infra_port += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let timestamps: Vec<f64> = tuple.sessions.iter().map(|s| s.start_ts).collect();
|
||||
if timestamps.len() < 20 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let beacon_score = pillar1::score(&BeaconingInput {
|
||||
timestamps: tuple.timestamps.clone(),
|
||||
timestamps: timestamps.clone(),
|
||||
proto: tuple.proto.clone(),
|
||||
})?;
|
||||
|
||||
if beacon_score.score >= 0.40 {
|
||||
let latest_ts = tuple.timestamps.last().copied().unwrap_or(0.0);
|
||||
let latest_ts = timestamps.last().copied().unwrap_or(0.0);
|
||||
let latest_date_str = match OffsetDateTime::from_unix_timestamp(latest_ts.floor() as i64)
|
||||
{
|
||||
Ok(dt) => {
|
||||
@@ -72,8 +157,8 @@ pub fn analyze_dir(
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
let raw_id = format!(
|
||||
"1|{}|{}|{}|{}",
|
||||
tuple.orig_h, tuple.resp_h, tuple.resp_p, latest_date_str
|
||||
"1|{}|{}|{}|{}|{}",
|
||||
tuple.proto, tuple.orig_h, tuple.resp_h, tuple.resp_p, latest_date_str
|
||||
);
|
||||
hasher.update(raw_id.as_bytes());
|
||||
let hash = hasher.finalize();
|
||||
@@ -84,9 +169,8 @@ pub fn analyze_dir(
|
||||
}
|
||||
|
||||
let summary = format!(
|
||||
"Beaconing detected: {}:{} -> {}:{} (period ~{:.0}s, score {:.3})",
|
||||
"Beaconing detected: {}:any -> {}:{} (period ~{:.0}s, score {:.3})",
|
||||
tuple.orig_h,
|
||||
"any",
|
||||
tuple.resp_h,
|
||||
tuple.resp_p,
|
||||
beacon_score.period_detected_seconds,
|
||||
@@ -96,37 +180,202 @@ pub fn analyze_dir(
|
||||
let evidence = serde_json::to_value(&beacon_score)?;
|
||||
let created_at = OffsetDateTime::now_utc().to_string();
|
||||
|
||||
alerts.push(Alert {
|
||||
id: alert_id,
|
||||
alert_items.push(AlertItem {
|
||||
pillar: 1,
|
||||
score: beacon_score.score,
|
||||
summary,
|
||||
evidence,
|
||||
created_at,
|
||||
orig_h: tuple.orig_h.clone(),
|
||||
dest: tuple.resp_h.clone(),
|
||||
resp_p: tuple.resp_p,
|
||||
alert: Alert {
|
||||
id: alert_id,
|
||||
pillar: 1,
|
||||
summary,
|
||||
score: beacon_score.score,
|
||||
evidence,
|
||||
created_at,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Pilar 2 evaluation: persistent TCP sessions
|
||||
for tuple in &tuple_sessions {
|
||||
if tuple.proto != "tcp" {
|
||||
continue;
|
||||
}
|
||||
|
||||
let candidates: Vec<SessionCandidate> = tuple
|
||||
.sessions
|
||||
.iter()
|
||||
.map(|s| SessionCandidate {
|
||||
start_ts: s.start_ts,
|
||||
end_ts: s.end_ts,
|
||||
duration: s.duration,
|
||||
parts: s.parts,
|
||||
orig_bytes: s.orig_bytes,
|
||||
resp_bytes: s.resp_bytes,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let session_score = pillar2::score(&SessionInput {
|
||||
sessions: candidates,
|
||||
failed_attempts: tuple.failed_attempts,
|
||||
})?;
|
||||
|
||||
if session_score.score >= 0.2181 {
|
||||
let mut hasher = Sha256::new();
|
||||
let raw_id = format!(
|
||||
"2|{}|{}|{}|{:.6}",
|
||||
tuple.orig_h, tuple.resp_h, tuple.resp_p, session_score.session_start_ts
|
||||
);
|
||||
hasher.update(raw_id.as_bytes());
|
||||
let hash = hasher.finalize();
|
||||
let mut alert_id = String::with_capacity(16);
|
||||
for b in &hash[..8] {
|
||||
use std::fmt::Write;
|
||||
let _ = write!(alert_id, "{b:02x}");
|
||||
}
|
||||
|
||||
let summary = format!(
|
||||
"Persistent session: {} -> {}:{} ({:.1}h, score {:.3})",
|
||||
tuple.orig_h,
|
||||
tuple.resp_h,
|
||||
tuple.resp_p,
|
||||
session_score.longest_session_seconds / 3600.0,
|
||||
session_score.score
|
||||
);
|
||||
|
||||
let evidence = serde_json::to_value(&session_score)?;
|
||||
let created_at = OffsetDateTime::now_utc().to_string();
|
||||
|
||||
alert_items.push(AlertItem {
|
||||
pillar: 2,
|
||||
orig_h: tuple.orig_h.clone(),
|
||||
dest: tuple.resp_h.clone(),
|
||||
resp_p: tuple.resp_p,
|
||||
alert: Alert {
|
||||
id: alert_id,
|
||||
pillar: 2,
|
||||
summary,
|
||||
score: session_score.score,
|
||||
evidence,
|
||||
created_at,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Pilar 3 evaluation: information volume in unique DNS subdomains per
|
||||
// (orig_h, base_domain), measured per UTC hour. JSD only travels in the
|
||||
// alert evidence with weight 0.0; it never changes the score.
|
||||
let dns_stats = store.ingest_dns_directory(directory)?;
|
||||
stats.files_seen += dns_stats.files_seen;
|
||||
stats.records_ingested += dns_stats.records_ingested;
|
||||
stats.lines_skipped += dns_stats.lines_skipped;
|
||||
|
||||
let dns_queries = store.query_window_dns()?;
|
||||
let protocol_scores = pillar3::score(&ProtocolInput {
|
||||
queries: dns_queries,
|
||||
})?;
|
||||
for protocol_score in &protocol_scores {
|
||||
if protocol_score.status == "ok" && protocol_score.score >= 0.40 {
|
||||
alert_items.push(AlertItem {
|
||||
pillar: 3,
|
||||
orig_h: protocol_score.orig_h.clone(),
|
||||
dest: protocol_score.base_domain.clone(),
|
||||
resp_p: 0,
|
||||
alert: pillar3::alert_for(protocol_score)?,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort alerts by (pillar, orig_h, dest, resp_p, id)
|
||||
alert_items.sort_by(|a, b| {
|
||||
(&a.pillar, &a.orig_h, &a.dest, a.resp_p, &a.alert.id).cmp(&(
|
||||
&b.pillar,
|
||||
&b.orig_h,
|
||||
&b.dest,
|
||||
b.resp_p,
|
||||
&b.alert.id,
|
||||
))
|
||||
});
|
||||
|
||||
let alerts: Vec<Alert> = alert_items.into_iter().map(|item| item.alert).collect();
|
||||
let hosts = crate::fusion::fuse(&alerts)?;
|
||||
|
||||
Ok(AnalysisReport {
|
||||
directory: directory.display().to_string(),
|
||||
logs_seen: stats.files_seen,
|
||||
dns_logs_seen: dns_stats.files_seen,
|
||||
records: stats.records_ingested,
|
||||
alerts,
|
||||
hosts,
|
||||
evidence_schema_version: 1,
|
||||
status: "ok",
|
||||
p1_excluded,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn classify_log(name: &str) -> ZeekLogKind {
|
||||
match name {
|
||||
"conn.log" => ZeekLogKind::Conn,
|
||||
"dns.log" => ZeekLogKind::Dns,
|
||||
"ssl.log" => ZeekLogKind::Ssl,
|
||||
"smb_mapping.log" => ZeekLogKind::SmbMapping,
|
||||
"dce_rpc.log" => ZeekLogKind::DceRpc,
|
||||
"rdp.log" => ZeekLogKind::Rdp,
|
||||
"ssh.log" => ZeekLogKind::Ssh,
|
||||
"kerberos.log" => ZeekLogKind::Kerberos,
|
||||
"ntlm.log" => ZeekLogKind::Ntlm,
|
||||
_ => ZeekLogKind::Unknown,
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn test_zeek_classify_logs() {
|
||||
assert_eq!(classify_log("conn.log"), ZeekLogKind::Conn);
|
||||
assert_eq!(classify_log("dns.log"), ZeekLogKind::Dns);
|
||||
assert_eq!(classify_log("ssl.log"), ZeekLogKind::Ssl);
|
||||
assert_eq!(classify_log("other.log"), ZeekLogKind::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_p1_exclusions_integration() {
|
||||
// Test 14: synthetic conn.log with 4 tuples:
|
||||
// 1. udp/123 (NTP) -> infra exclusion
|
||||
// 2. 224.0.0.251:5353 (multicast) -> multicast exclusion
|
||||
// 3. icmp -> non-tcp/udp exclusion
|
||||
// 4. tcp/8443 -> valid beaconing tuple (alerts)
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let log_dir = temp_dir.path().join("logs");
|
||||
let data_dir = temp_dir.path().join("data");
|
||||
fs::create_dir_all(&log_dir).unwrap();
|
||||
fs::create_dir_all(&data_dir).unwrap();
|
||||
|
||||
let mut lines = Vec::new();
|
||||
lines.push("#separator \\x09".to_string());
|
||||
lines.push(
|
||||
"#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto".to_string(),
|
||||
);
|
||||
|
||||
let t0 = 1793400000.0;
|
||||
for i in 0..42 {
|
||||
let t = t0 + (i as f64 * 14400.0);
|
||||
// 1. udp/123
|
||||
lines.push(format!(
|
||||
"{t:.6}\tu1_{i}\t10.0.0.1\t123\t198.51.100.1\t123\tudp"
|
||||
));
|
||||
// 2. multicast
|
||||
lines.push(format!(
|
||||
"{t:.6}\tu2_{i}\t10.0.0.1\t5353\t224.0.0.251\t5353\tudp"
|
||||
));
|
||||
// 3. icmp
|
||||
lines.push(format!(
|
||||
"{t:.6}\tu3_{i}\t10.0.0.1\t0\t198.51.100.2\t0\ticmp"
|
||||
));
|
||||
// 4. tcp/8443
|
||||
lines.push(format!(
|
||||
"{t:.6}\tu4_{i}\t10.0.0.1\t40000\t198.51.100.3\t8443\ttcp"
|
||||
));
|
||||
}
|
||||
|
||||
let conn_file = log_dir.join("conn.log");
|
||||
fs::write(&conn_file, lines.join("\n")).unwrap();
|
||||
|
||||
let report = analyze_dir(&log_dir, &data_dir).unwrap();
|
||||
assert_eq!(report.p1_excluded.infra_port, 1);
|
||||
assert_eq!(report.p1_excluded.multicast_broadcast, 1);
|
||||
assert_eq!(report.p1_excluded.non_tcp_udp, 1);
|
||||
assert_eq!(report.alerts.len(), 1, "only tcp/8443 should alert");
|
||||
assert!(report.alerts[0].summary.contains("8443"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
Synthetic test fixture for Zeek idle TCP split verification.
|
||||
|
||||
Generated with:
|
||||
1. python3 research/validation/mkpcap_idle.py /tmp/zeek_split_gen/pcap/idle.pcap
|
||||
2. docker run --rm --user "$(id -u):$(id -g)" \
|
||||
-v /tmp/zeek_split_gen/pcap:/pcap:ro -v /tmp/zeek_split_gen/out:/out -w /out \
|
||||
zeek/zeek:9.0.1 zeek -C -r /pcap/idle.pcap
|
||||
|
||||
Contains one TCP session (10.0.0.5:40000 -> 203.0.113.9:443) split by Zeek 9.0.1 across a 600s idle gap into two conn.log lines:
|
||||
- First line: ts=1793400000.000000, duration=1741.050000, history=ShADd
|
||||
- Second line: ts=1793402401.000000, duration=1800.050000, history=DdFfA (no 'S' in history, gap = 659.95s <= 3600s)
|
||||
|
||||
Expected chained session: L = 4201.05s, parts = 2.
|
||||
@@ -0,0 +1,11 @@
|
||||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path conn
|
||||
#open 2026-11-01-00-28-56
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents ip_proto
|
||||
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] count
|
||||
1793400000.000000 Cr5nhu4rvbFFhLnlC7 10.0.0.5 40000 203.0.113.9 443 tcp - 1741.050000 600 1200 S1 T T 0 ShADd 32 1880 31 2440 - 6
|
||||
1793402401.000000 Cn4avD2d83MCmnH4xl 10.0.0.5 40000 203.0.113.9 443 tcp - 1800.050000 600 1200 SF T T 0 DdFfA 32 1880 31 2440 - 6
|
||||
#close 2026-11-01-00-28-56
|
||||
@@ -0,0 +1,179 @@
|
||||
//! End-to-end Pilar 3: a synthetic Zeek `dns.log` is ingested into DuckDB,
|
||||
//! read back through the 7-day window and scored by `pillar3`. The tunnel
|
||||
//! names are synthetic (SplitMix64 base32), never real captures.
|
||||
|
||||
use acaua_core::{pillar3, storage::Storage};
|
||||
use std::fs;
|
||||
|
||||
struct SplitMix64 {
|
||||
state: u64,
|
||||
}
|
||||
|
||||
impl SplitMix64 {
|
||||
fn new(seed: u64) -> Self {
|
||||
Self { state: seed }
|
||||
}
|
||||
fn next_u64(&mut self) -> u64 {
|
||||
self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = self.state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
fn below(&mut self, n: u64) -> u64 {
|
||||
self.next_u64() % n
|
||||
}
|
||||
}
|
||||
|
||||
const HEADER: &str = "#separator \\x09\n\
|
||||
#set_separator\t,\n\
|
||||
#empty_field\t(empty)\n\
|
||||
#unset_field\t-\n\
|
||||
#path\tdns\n\
|
||||
#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tproto\ttrans_id\trtt\tquery\tqclass\tqclass_name\tqtype\tqtype_name\trcode\trcode_name\tAA\tTC\tRD\tRA\tZ\tanswers\tTTLs\trejected\topcode\topcode_name\n";
|
||||
|
||||
fn dns_line(ts: f64, uid: &str, host: &str, port: u16, query: &str) -> String {
|
||||
format!(
|
||||
"{ts:.6}\t{uid}\t{host}\t{port}\t10.0.2.3\t53\tudp\t1\t0.001\t{query}\t1\tC_INET\t16\tTXT\t0\tNOERROR\tF\tF\tT\tT\t0\t-\t-\tF\t0\tQUERY\n"
|
||||
)
|
||||
}
|
||||
|
||||
fn base32_name(rng: &mut SplitMix64) -> String {
|
||||
const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567";
|
||||
let labels = 2 + rng.below(2) as usize;
|
||||
let mut parts = Vec::new();
|
||||
for _ in 0..labels {
|
||||
let len = 50 + rng.below(14) as usize;
|
||||
parts.push(
|
||||
(0..len)
|
||||
.map(|_| ALPHABET[rng.below(32) as usize] as char)
|
||||
.collect::<String>(),
|
||||
);
|
||||
}
|
||||
parts.join(".")
|
||||
}
|
||||
|
||||
fn write_dns_log(dir: &std::path::Path, lines: &[String]) {
|
||||
let mut log = String::from(HEADER);
|
||||
for line in lines {
|
||||
log.push_str(line);
|
||||
}
|
||||
fs::write(dir.join("dns.log"), log).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::inconsistent_digit_grouping)]
|
||||
fn synthetic_tunnel_is_ingested_and_alerted() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let logs = temp.path().join("logs");
|
||||
fs::create_dir_all(&logs).unwrap();
|
||||
let store = Storage::open(temp.path().join("data")).unwrap();
|
||||
|
||||
let mut rng = SplitMix64::new(2026_10_31);
|
||||
let mut lines = Vec::new();
|
||||
for i in 0..100 {
|
||||
lines.push(dns_line(
|
||||
1_800_000_000.0 + i as f64 * 30.0,
|
||||
&format!("C{i}"),
|
||||
"10.66.66.66",
|
||||
50000 + i,
|
||||
&format!("{}.t.example-tunnel.net", base32_name(&mut rng)),
|
||||
));
|
||||
}
|
||||
write_dns_log(&logs, &lines);
|
||||
|
||||
let stats = store.ingest_dns_directory(&logs).unwrap();
|
||||
assert_eq!(stats.files_seen, 1);
|
||||
assert_eq!(stats.records_ingested, 100);
|
||||
|
||||
let queries = store.query_window_dns().unwrap();
|
||||
assert_eq!(queries.len(), 100);
|
||||
|
||||
let scores = pillar3::score(&pillar3::ProtocolInput { queries }).unwrap();
|
||||
assert_eq!(scores.len(), 1);
|
||||
let score = &scores[0];
|
||||
assert_eq!(score.orig_h, "10.66.66.66");
|
||||
assert_eq!(score.base_domain, "example-tunnel.net");
|
||||
assert_eq!(score.status, "ok");
|
||||
assert!(
|
||||
score.score >= 0.40,
|
||||
"score {} bits {}",
|
||||
score.score,
|
||||
score.information_bits
|
||||
);
|
||||
assert_eq!(score.weights.information, 1.0);
|
||||
assert_eq!(score.weights.jsd, 0.0);
|
||||
assert_eq!(score.jsd_reference_groups, 0);
|
||||
assert_eq!(score.jsd, 0.0);
|
||||
|
||||
let alerts = pillar3::alerts(&scores).unwrap();
|
||||
assert_eq!(alerts.len(), 1);
|
||||
assert_eq!(alerts[0].pillar, 3);
|
||||
assert_eq!(alerts[0].id.len(), 16);
|
||||
assert_eq!(alerts[0].score, score.score);
|
||||
let evidence: serde_json::Value = alerts[0].evidence.clone();
|
||||
assert_eq!(evidence["weights"]["jsd"].as_f64(), Some(0.0));
|
||||
assert_eq!(evidence["weights"]["information"].as_f64(), Some(1.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthetic_cloudfront_stays_below_the_alert_threshold() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let logs = temp.path().join("logs");
|
||||
fs::create_dir_all(&logs).unwrap();
|
||||
let store = Storage::open(temp.path().join("data")).unwrap();
|
||||
|
||||
let mut rng = SplitMix64::new(7);
|
||||
let mut lines = Vec::new();
|
||||
const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789";
|
||||
for i in 0..45 {
|
||||
let name: String = (0..14)
|
||||
.map(|_| ALPHABET[rng.below(ALPHABET.len() as u64) as usize] as char)
|
||||
.collect();
|
||||
lines.push(dns_line(
|
||||
1_800_000_000.0 + i as f64 * 60.0,
|
||||
&format!("C{i}"),
|
||||
"10.0.2.15",
|
||||
41000 + i,
|
||||
&format!("{name}.cloudfront.net"),
|
||||
));
|
||||
}
|
||||
write_dns_log(&logs, &lines);
|
||||
store.ingest_dns_directory(&logs).unwrap();
|
||||
|
||||
let queries = store.query_window_dns().unwrap();
|
||||
let scores = pillar3::score(&pillar3::ProtocolInput { queries }).unwrap();
|
||||
let alerts = pillar3::alerts(&scores).unwrap();
|
||||
assert!(alerts.is_empty());
|
||||
assert_eq!(scores.len(), 1);
|
||||
assert_eq!(scores[0].score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analyze_dir_emits_a_pillar3_alert_without_any_conn_log() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let logs = temp.path().join("logs");
|
||||
let data = temp.path().join("data");
|
||||
fs::create_dir_all(&logs).unwrap();
|
||||
|
||||
let mut rng = SplitMix64::new(99);
|
||||
let mut lines = Vec::new();
|
||||
for i in 0..100 {
|
||||
lines.push(dns_line(
|
||||
1_800_000_000.0 + i as f64 * 30.0,
|
||||
&format!("C{i}"),
|
||||
"10.66.66.68",
|
||||
52000 + i,
|
||||
&format!("{}.t.example-tunnel.net", base32_name(&mut rng)),
|
||||
));
|
||||
}
|
||||
write_dns_log(&logs, &lines);
|
||||
|
||||
let report = acaua_core::zeek::analyze_dir(&logs, &data).unwrap();
|
||||
assert_eq!(report.alerts.len(), 1);
|
||||
assert_eq!(report.alerts[0].pillar, 3);
|
||||
assert!(report.alerts[0]
|
||||
.summary
|
||||
.contains("High-information DNS subdomains"));
|
||||
assert_eq!(report.alerts[0].id.len(), 16);
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use acaua_core::{doctor, fusion, pillar1, pillar2, pillar3, pillar4, storage, ze
|
||||
fn pillar1_returns_a_skeleton_score() {
|
||||
let input = pillar1::BeaconingInput {
|
||||
timestamps: vec![1.0, 2.0, 3.0],
|
||||
proto: "tcp".to_string(),
|
||||
};
|
||||
let score = pillar1::score(&input).unwrap();
|
||||
assert_eq!(score.observations, 3);
|
||||
@@ -13,33 +14,49 @@ fn pillar1_returns_a_skeleton_score() {
|
||||
#[test]
|
||||
fn pillar2_counts_long_connections_without_panicking() {
|
||||
let input = pillar2::SessionInput {
|
||||
durations_secs: vec![1.0, 400.0],
|
||||
bytes: vec![10, 20],
|
||||
sessions: vec![
|
||||
pillar2::SessionCandidate {
|
||||
start_ts: 1000.0,
|
||||
end_ts: 1001.0,
|
||||
duration: 1.0,
|
||||
parts: 1,
|
||||
orig_bytes: 10,
|
||||
resp_bytes: 20,
|
||||
},
|
||||
pillar2::SessionCandidate {
|
||||
start_ts: 2000.0,
|
||||
end_ts: 2400.0,
|
||||
duration: 400.0,
|
||||
parts: 1,
|
||||
orig_bytes: 100,
|
||||
resp_bytes: 200,
|
||||
},
|
||||
],
|
||||
failed_attempts: 0,
|
||||
};
|
||||
let score = pillar2::score(&input).unwrap();
|
||||
assert_eq!(score.long_connections, 1);
|
||||
assert_eq!(score.status, "skeleton");
|
||||
assert_eq!(score.longest_session_seconds, 400.0);
|
||||
assert_eq!(score.status, "ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pillar3_handles_an_empty_protocol_list() {
|
||||
let score = pillar3::score(&pillar3::ProtocolInput::default()).unwrap();
|
||||
assert_eq!(score.distinct_protocols, 0);
|
||||
assert_eq!(score.status, "skeleton");
|
||||
fn pillar3_handles_empty_dns_input() {
|
||||
let scores = pillar3::score(&pillar3::ProtocolInput::default()).unwrap();
|
||||
assert!(scores.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pillar4_fuses_three_inputs() {
|
||||
let score = pillar4::score(&pillar4::FusionInput::default()).unwrap();
|
||||
assert_eq!(score.inputs, 3);
|
||||
assert_eq!(score.status, "skeleton");
|
||||
let score = pillar4::score("10.0.0.1", &pillar4::FusionInput::default()).unwrap();
|
||||
assert_eq!(score.terms.len(), 3);
|
||||
assert_eq!(score.formula_version, "1.0");
|
||||
assert_eq!(score.score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fusion_returns_no_alerts_yet() {
|
||||
let report = fusion::fuse(&[0.1, 0.2]).unwrap();
|
||||
assert!(report.alerts.is_empty());
|
||||
assert_eq!(report.status, "skeleton");
|
||||
let hosts = fusion::fuse(&[]).unwrap();
|
||||
assert!(hosts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -67,5 +84,5 @@ fn zeek_classifies_known_logs() {
|
||||
fn doctor_reports_checks() {
|
||||
let report = doctor::run();
|
||||
assert!(!report.checks.is_empty());
|
||||
assert_eq!(report.status, "skeleton");
|
||||
assert!(report.status == "ok" || report.status == "degraded");
|
||||
}
|
||||
Reference in new issue
Block a user