Lucas RayanandClaude Sonnet 5
c64bbbf775
Pillar 1 v1.1 (shipped earlier) flagged benign browsing bursts as beaconing: on short captures, grid periods longer than the observed span never complete a cycle, so significance saturates to 1 and the score sticks at exactly 0.5. v1.2 caps the grid at a third of the observed span, requires a minimum number of completed cycles, and excludes protocols that are periodic by nature (NTP, DNS, mDNS, ICMP). Validated against 300 seeds per scenario: false positives on bursty traffic drop from the 290-300/300 that v1.1 produced to 0/300, with every beacon-detection scenario scoring unchanged. Add Pillar 2 (persistent/long-lived sessions, with Zeek's fragmented TCP sessions rejoined after a 5-minute gap), Pillar 3 (DNS tunneling via Shannon-entropy information volume per subdomain per hour; the Jensen-Shannon divergence was measured and does not separate tunnels from benign CDN subdomains, so it stays in the evidence at zero weight instead of in the score), and Pillar 4 (deterministic noisy-OR fusion across the three pillars). The fusion formula was validated against 8 labeled real-traffic vectors (IoT-23, CTU-Normal) with tolerance <= 1e-4. A known limitation is kept, not hidden: a benign Philips HUE bulb's strict periodic polling and a 21.9h session score it above three real C2 hosts. No weight or threshold was tuned to mask this; it is documented in the architecture docs instead. Also fixes real bugs found along the way: ingest_dir's partition count summed file names instead of distinct dates, DuckDB errors were counted as bad rows, /api/alerts always returned an empty list regardless of the last analysis, and the dashboard still claimed every pillar was unimplemented. Restores four design corrections that were missing from docs/architecture.md (KL to Jensen-Shannon, the persistent-session detector, the JA4 licensing boundary, and deterministic reduction order), and catalogs the public datasets used for validation (IoT-23, CTU-Normal, Mendeley DTQBC) with their licenses and SHA-256 checksums under research/validation/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VAyr8d7HuAdUGB6aA5xmDD
46 lines
2.6 KiB
Python
46 lines
2.6 KiB
Python
# Applies Pillar 1 v1.1 and v1.2 (and ablations) to real Zeek conn.log files; tuple label from IoT-23 labeled file.
|
|
import sys, os, collections, importlib.util
|
|
here = os.path.dirname(os.path.abspath(__file__))
|
|
spec = importlib.util.spec_from_file_location('sim', os.path.join(here, 'p1v12sim_lib.py')); sim = importlib.util.module_from_spec(spec); spec.loader.exec_module(sim)
|
|
def read(path):
|
|
fields = None
|
|
for line in open(path, errors='replace'):
|
|
line = line.rstrip('\n')
|
|
if line.startswith('#fields'): fields = line.split('\t')[1:]; continue
|
|
if line.startswith('#') or not line: continue
|
|
yield dict(zip(fields, line.split('\t')))
|
|
def labels(path):
|
|
lab = {}
|
|
if not path or not os.path.exists(path): return lab
|
|
for r in read(path):
|
|
last = ' '.join(v for k, v in r.items() if k in ('tunnel_parents', 'label', 'detailed-label', 'det_label'))
|
|
mal = 'Malicious' in last or 'Malicious' in str(list(r.values())[-3:])
|
|
k = (r['id.orig_h'], r['id.resp_h'], r['id.resp_p'])
|
|
lab[k] = lab.get(k, False) or mal
|
|
return lab
|
|
import ipaddress
|
|
EXCL = {('udp','5355'),('udp','546'),('udp','547'),('udp', '123'), ('udp', '53'), ('tcp', '53'), ('udp', '5353'), ('udp', '137'), ('udp', '138'), ('udp', '67'), ('udp', '68'), ('udp', '1900')}
|
|
conn, labf = sys.argv[1], (sys.argv[2] if len(sys.argv) > 2 else None)
|
|
lab = labels(labf)
|
|
tup = collections.defaultdict(list); protos = {}
|
|
for r in read(conn):
|
|
k = (r['id.orig_h'], r['id.resp_h'], r['id.resp_p']); tup[k].append(float(r['ts'])); protos.setdefault(k, set()).add(r['proto'])
|
|
rows = []
|
|
for k, ts in sorted(tup.items()):
|
|
if len(ts) < 20: continue
|
|
a = sim.v11(ts)[0]; b = sim.v12(ts)
|
|
span_only = sim.v12(ts, gate=1e-12)[0]; gate_only = sim.v12(ts, cycles=1e-12)[0]
|
|
try:
|
|
a_ = ipaddress.ip_address(k[1]); mc = a_.is_multicast or k[1] == '255.255.255.255'
|
|
except ValueError:
|
|
mc = False
|
|
excl = mc or any(p not in ('tcp', 'udp') for p in protos[k]) or any((p, k[2]) in EXCL for p in protos[k])
|
|
rows.append((k, len(ts), a, span_only, gate_only, b[0], b[1], b[2], excl, lab.get(k)))
|
|
print(f"== {conn}: tuples with n>=20: {len(rows)}")
|
|
print("tuple n v1.1 span_only gate_only v1.2 status period excluded label")
|
|
for r in rows:
|
|
if max(r[2], r[5]) >= .40:
|
|
print(f" {r[0]} n={r[1]} v11={r[2]:.3f} span={r[3]:.3f} gate={r[4]:.3f} v12={r[5]:.3f} {r[6]} per={r[7] and round(r[7])} excl={r[8]} mal={r[9]}")
|
|
c = lambda i, ex: sum(1 for r in rows if r[i] >= .4 and (not ex or not r[8]))
|
|
print(f" alerts>=0.40: v1.1={c(2,False)} span_only={c(3,False)} gate_only={c(4,False)} v1.2={c(5,False)} v1.2+exclusion={c(5,True)}")
|