From 7f00445d679517c5a9c20615fbe1be7b2870095c Mon Sep 17 00:00:00 2001 From: Lucas Rayan Date: Sun, 18 Oct 2026 23:46:47 -0300 Subject: [PATCH] =?UTF-8?q?Carving=20de=20JPEG=20fragmentado:=20decodifica?= =?UTF-8?q?dor=20de=20entropia,=20=C3=ADndice=20de=20clusters=20e=20remont?= =?UTF-8?q?agem=20de=20at=C3=A9=20tr=C3=AAs=20fragmentos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/guara-core/src/carve.rs | 15 +- crates/guara-core/src/defrag.rs | 332 ++++++++++++++ crates/guara-core/src/fs/ext.rs | 9 +- crates/guara-core/src/jpegscan.rs | 598 ++++++++++++++++++++++++++ crates/guara-core/src/lib.rs | 2 + crates/guara-core/src/scan.rs | 78 +++- crates/guara-core/src/service.rs | 33 +- crates/guara-core/tests/common/mod.rs | 6 + crates/guara-core/tests/defrag.rs | 183 ++++++++ crates/guara-core/tests/jpegscan.rs | 70 +++ 10 files changed, 1304 insertions(+), 22 deletions(-) create mode 100644 crates/guara-core/src/defrag.rs create mode 100644 crates/guara-core/src/jpegscan.rs create mode 100644 crates/guara-core/tests/defrag.rs create mode 100644 crates/guara-core/tests/jpegscan.rs diff --git a/crates/guara-core/src/carve.rs b/crates/guara-core/src/carve.rs index c894f57..2f2397c 100644 --- a/crates/guara-core/src/carve.rs +++ b/crates/guara-core/src/carve.rs @@ -62,6 +62,7 @@ pub fn carve_range( break; } let scan_len = got.min(CHUNK).min((end - pos) as usize); + ev.chunk(pos, &buf[..scan_len]); let mut i = 0usize; while i < scan_len { let abs = pos + i as u64; @@ -78,10 +79,12 @@ pub fn carve_range( || cfg.categories.contains(&category_for_ext(hit.ext)); if hit.len >= MIN_LEN && allowed { let action = ev.hit(abs, &hit); - if matches!(action, HitAction::Skip) - && hit.confidence != crate::types::Confidence::Low - { - skip_until = abs + hit.len; + match action { + HitAction::Skip if hit.confidence != crate::types::Confidence::Low => { + skip_until = abs + hit.len; + } + HitAction::SkipLen(n) => skip_until = abs + n, + _ => {} } } } @@ -109,6 +112,8 @@ pub enum CarveEnd { /// Receptor de eventos do carving. pub trait CarveEvents { fn advance(&mut self, bytes: u64); + /// Cada trecho lido, antes de procurar assinaturas (para indexar o que serve a recomposição). + fn chunk(&mut self, _abs: u64, _data: &[u8]) {} fn hit(&mut self, offset: u64, hit: &Hit) -> HitAction; } @@ -116,6 +121,8 @@ pub trait CarveEvents { pub enum HitAction { /// Pula o conteúdo do arquivo reconhecido (comportamento padrão). Skip, + /// Pula só estes bytes (arquivo cujo resto não é confiável). + SkipLen(u64), /// Segue setor a setor (acerto descartado). Continue, } diff --git a/crates/guara-core/src/defrag.rs b/crates/guara-core/src/defrag.rs new file mode 100644 index 0000000..fea0a95 --- /dev/null +++ b/crates/guara-core/src/defrag.rs @@ -0,0 +1,332 @@ +//! Recomposição de arquivos fragmentados no carving. +//! +//! Para JPEG: valida os bytes contíguos com o decodificador de entropia; se a imagem quebra, +//! procura entre os clusters "com cara de JPEG" próximos a continuação cujo DC de luminância +//! emenda sem salto, até três fragmentos. É o que dá para fazer sem tabela de alocação, e só +//! vale quando o sistema de arquivos não deixou metadados para o arquivo. + +use crate::jpegscan::{self, Header, State, Stop}; +use crate::source::{read_vec, Source}; +use crate::types::Extent; + +const MAX_FILE: usize = 48 << 20; +/// Janela de busca da continuação, em bytes, depois e antes do primeiro fragmento. +const WINDOW_AFTER: u64 = 512 << 20; +const WINDOW_BEFORE: u64 = 64 << 20; +const PROBE: usize = 8192; +pub const CLUSTER: u64 = 4096; +const MAX_PROBES: usize = 20_000; + +/// Marca os clusters (de 4 KiB, alinhados ao início da origem) que parecem dados entropy-coded +/// de JPEG: sem marcadores no meio e com bytes bem distribuídos. +#[derive(Default, Clone)] +pub struct ClusterIndex { + bits: Vec, +} + +impl ClusterIndex { + pub fn new(size: u64) -> Self { + Self { bits: vec![0; (size / CLUSTER / 64 + 1) as usize] } + } + pub fn set(&mut self, off: u64) { + let c = (off / CLUSTER) as usize; + if let Some(w) = self.bits.get_mut(c / 64) { + *w |= 1 << (c % 64); + } + } + pub fn get(&self, off: u64) -> bool { + let c = (off / CLUSTER) as usize; + self.bits.get(c / 64).map(|w| w >> (c % 64) & 1 == 1).unwrap_or(false) + } + /// Examina um trecho lido e marca os clusters candidatos. + pub fn scan_chunk(&mut self, abs: u64, buf: &[u8]) { + let first = abs.div_ceil(CLUSTER) * CLUSTER; + let mut o = first; + while o + CLUSTER <= abs + buf.len() as u64 { + let c = &buf[(o - abs) as usize..(o - abs + CLUSTER) as usize]; + if jpeg_like(c) { + self.set(o); + } + o += CLUSTER; + } + } + pub fn iter_range(&self, from: u64, to: u64) -> impl Iterator + '_ { + let (a, b) = (from.div_ceil(CLUSTER), to / CLUSTER); + (a..b).filter(move |&c| self.bits.get(c as usize / 64).map(|w| w >> (c % 64) & 1 == 1).unwrap_or(false)).map(|c| c * CLUSTER) + } +} + +pub fn jpeg_like(c: &[u8]) -> bool { + let mut seen = [false; 256]; + let mut distinct = 0; + for (i, &b) in c.iter().enumerate() { + if !seen[b as usize] { + seen[b as usize] = true; + distinct += 1; + } + if b == 0xFF { + match c.get(i + 1) { + Some(&n) if n == 0x00 || (0xD0..=0xD7).contains(&n) => {} + Some(0xD9) if i + 2 >= c.len() - 16 => {} + None => {} + _ => return false, + } + } + } + distinct >= 190 +} + +pub struct Pool<'a> { + pub index: &'a ClusterIndex, + /// Extents já atribuídos a outros arquivos. + pub used: &'a [Extent], +} + +impl Pool<'_> { + fn free_at(&self, off: u64, len: u64) -> bool { + !self.used.iter().any(|u| off < u.end() && off + len > u.offset) + } +} + +#[derive(Debug)] +pub enum Analysis { + /// Contíguo e consistente; `len` inclui o EOI. + Intact { len: u64 }, + /// Remontado a partir de fragmentos. + Rebuilt { extents: Vec, size: u64 }, + /// Só o início é confiável. + Broken { good_len: u64 }, + /// Formato que não sabemos validar (progressivo etc.). + Unsupported, +} + +/// Procura FF D9 logo depois do último MCU (até 64 bytes de preenchimento). +fn eoi_after(data: &[u8], pos: usize) -> Option { + let end = (pos + 64).min(data.len().saturating_sub(1)); + (pos.saturating_sub(2)..end).find(|&i| data[i] == 0xFF && data[i + 1] == 0xD9).map(|i| i + 2) +} + +/// Valida o JPEG que começa em `off` e, se estiver quebrado, tenta recompô-lo. +pub fn analyze_jpeg(src: &dyn Source, off: u64, hit_len: u64, pool: &Pool, allow_rebuild: bool) -> Analysis { + let read_len = (hit_len as usize).clamp(4096, MAX_FILE); + let Ok(buf) = read_vec(src, off, read_len) else { return Analysis::Unsupported }; + let Some(h) = jpegscan::parse_header(&buf) else { return Analysis::Unsupported }; + let d = jpegscan::decode(&h, &buf, State::start(&h), u32::MAX, 512); + let disc = jpegscan::first_discontinuity(&h, &d.dc); + if d.stop == Stop::Done && disc.is_none() { + if let Some(end) = eoi_after(&buf, d.state.pos) { + return Analysis::Intact { len: end as u64 }; + } + } + let mut stop_pos = match d.stop { + Stop::Marker(_, p) | Stop::Error(p) => p, + Stop::Truncated | Stop::Done => buf.len(), + }; + if let Some(m) = disc { + let bad = d.snaps.iter().rev().find(|s| (s.mcu as usize) <= m).map(|s| s.pos).unwrap_or(h.scan_start); + stop_pos = stop_pos.min(bad); + } + let cl = CLUSTER as usize; + if off % CLUSTER != 0 || !allow_rebuild { + return Analysis::Broken { good_len: (stop_pos / 512 * 512).max(512) as u64 }; + } + + let mut pieces: Vec<(u64, usize)> = vec![]; + let mut joined = buf; + let mut cur_base = off; // início absoluto do fragmento atual + let mut frag_start = 0usize; // posição em `joined` onde o fragmento atual começa + let mut snaps = d.snaps; + let mut suspect = stop_pos; + + for _ in 0..3 { + let top = suspect.saturating_sub(frag_start) / cl * cl; + let mut best: Option<(usize, u64, State, f64)> = None; + let mut probes = 0usize; + for back in 0..4usize { + let Some(bc_rel) = top.checked_sub(back * cl).filter(|&b| b > 0) else { break }; + let bc = frag_start + bc_rel; + let Some(snap) = snaps + .iter() + .rev() + .find(|s| s.pos <= bc) + .cloned() + .or_else(|| (frag_start == 0).then(|| State::start(&h))) + else { + continue; + }; + let mut plane = jpegscan::Plane::new(&h); + let _ = jpegscan::decode_plane(&h, &joined[..bc], State::start(&h), u32::MAX, &mut plane); + let lo = off.saturating_sub(WINDOW_BEFORE); + let hi = cur_base + bc_rel as u64 + WINDOW_AFTER; + let pre = jpegscan::decode(&h, &joined[..bc], State::start(&h), u32::MAX, 0); + let bpm_prefix = (pre.state.pos.saturating_sub(h.scan_start)) as f64 / pre.dc.len().max(1) as f64; + let mut ranked: Vec<(f64, u64)> = vec![]; + for q in pool.index.iter_range(lo, hi) { + if q >= cur_base && q < cur_base + bc_rel as u64 + CLUSTER { + continue; + } + if !pool.free_at(q, CLUSTER) || pieces.iter().any(|&(po, pl)| q < po + pl as u64 && q + CLUSTER > po) { + continue; + } + probes += 1; + if probes > MAX_PROBES { + break; + } + if let Some(sc) = probe_candidate(src, &h, &joined[..bc], &plane, &snap, q, bpm_prefix) { + ranked.push((sc, q)); + } + } + ranked.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + ranked.truncate(120); + // Etapa 2: a continuação certa fecha a imagem exatamente no último MCU. + let mut longest: Option<(usize, u64, f64)> = None; + for (sc, q) in ranked { + let avail = (src.size() - q) as usize; + let more = read_vec(src, q, MAX_FILE.min(avail)).unwrap_or_default(); + let mut j2 = joined[..bc].to_vec(); + j2.extend_from_slice(&more); + let d2 = jpegscan::decode(&h, &j2, snap.clone(), u32::MAX, 0); + if d2.stop == Stop::Done && eoi_after(&j2, d2.state.pos).is_some() { + best = Some((bc, q, snap.clone(), 0.0)); + break; + } + let reach = match d2.stop { + Stop::Marker(_, p) | Stop::Error(p) => p.saturating_sub(bc), + _ => j2.len() - bc, + }; + if reach >= 3 * cl && longest.map_or(true, |l| sc < l.2) { + longest = Some((reach, q, sc)); + } + } + if best.is_none() { + if let Some((_, q, sc)) = longest { + if sc < 25.0 { + let cand = (bc, q, snap.clone(), sc); + if best.as_ref().map_or(true, |b| sc < b.3) { + best = Some(cand); + } + } + } + } + if best.as_ref().map_or(false, |b| b.3 == 0.0) { + break; + } + } + let Some((bc, q, snap, _)) = best else { break }; + pieces.push((cur_base, bc - frag_start)); + let more = read_vec(src, q, MAX_FILE.min((src.size() - q) as usize)).unwrap_or_default(); + joined.truncate(bc); + joined.extend_from_slice(&more); + let d2 = jpegscan::decode(&h, &joined, snap, u32::MAX, 512); + if d2.stop == Stop::Done { + if let Some(end) = eoi_after(&joined, d2.state.pos) { + pieces.push((q, end - bc)); + return finish(pieces); + } + } + // Quebrou de novo: onde? + let mut sp = match d2.stop { + Stop::Marker(_, p) | Stop::Error(p) => p, + _ => joined.len(), + }; + let all = jpegscan::decode(&h, &joined, State::start(&h), u32::MAX, 0); + if let Some(m) = jpegscan::first_discontinuity(&h, &all.dc) { + let bad = d2.snaps.iter().rev().find(|s| (s.mcu as usize) <= m).map(|s| s.pos).unwrap_or(bc); + if bad > bc { + sp = sp.min(bad); + } + } + snaps = d2.snaps; + cur_base = q; + frag_start = bc; + suspect = sp.max(bc + cl); + } + let good = pieces.iter().map(|p| p.1).sum::() + suspect.saturating_sub(frag_start).min(cl * 4) / 512 * 512; + Analysis::Broken { good_len: (stop_pos.min(good.max(512)) / 512 * 512).max(512) as u64 } +} + +fn finish(pieces: Vec<(u64, usize)>) -> Analysis { + let mut extents: Vec = vec![]; + for (o, l) in &pieces { + match extents.last_mut() { + Some(last) if last.end() == *o => last.len += *l as u64, + _ => extents.push(Extent::new(*o, *l as u64)), + } + } + let size = pieces.iter().map(|p| p.1 as u64).sum(); + if extents.len() == 1 { + return Analysis::Intact { len: size }; + } + Analysis::Rebuilt { extents, size } +} + +/// Pontuação (menor é melhor) de `q` como continuação de `prefix`: diferença média de pixels +/// nas bordas entre os MCUs novos e seus vizinhos. Natural: poucos níveis de cinza; lixo: dezenas. +fn probe_candidate(src: &dyn Source, h: &Header, prefix: &[u8], plane: &jpegscan::Plane, snap: &State, q: u64, bpm_prefix: f64) -> Option { + let more = read_vec(src, q, PROBE).ok()?; + if more.len() < 1024 { + return None; + } + let mut joined = Vec::with_capacity(prefix.len() + more.len()); + joined.extend_from_slice(prefix); + joined.extend_from_slice(&more); + let seam = plane.have; + const K: u32 = 36; + let mut news: std::collections::HashMap> = std::collections::HashMap::new(); + let mut hook = |mcu: u32, blocks: &[[i32; 64]]| { + if mcu >= seam { + news.insert(mcu, jpegscan::mcu_luma(h, blocks)); + } + }; + let want = seam.saturating_sub(snap.mcu) + K + 1; + let d = jpegscan::decode_hook(h, &joined, snap.clone(), want, 0, Some(&mut hook)); + match d.stop { + Stop::Error(_) => return None, + Stop::Marker(m, _) if m != 0xD9 => return None, + _ => {} + } + let (mw, mh) = (plane.mw, plane.mh); + let mx = h.mcus_x; + let get = |idx: u32| -> Option> { + if idx < seam { + let (x, y) = ((idx % mx) as usize, (idx / mx) as usize); + let mut v = vec![0u8; mw * mh]; + for r in 0..mh { + let o = (y * mh + r) * plane.w + x * mw; + v[r * mw..(r + 1) * mw].copy_from_slice(&plane.px[o..o + mw]); + } + Some(v) + } else { + news.get(&idx).cloned() + } + }; + let (mut sum, mut n) = (0f64, 0u32); + for idx in seam + 1..seam + 1 + K { + let Some(cur) = news.get(&idx) else { break }; + if idx % mx > 0 { + if let Some(l) = get(idx - 1) { + for y in 0..mh { + sum += (cur[y * mw] as i32 - l[y * mw + mw - 1] as i32).abs() as f64; + n += 1; + } + } + } + if idx >= mx { + if let Some(t) = get(idx - mx) { + for x in 0..mw { + sum += (cur[x] as i32 - t[(mh - 1) * mw + x] as i32).abs() as f64; + n += 1; + } + } + } + } + if n < 40 { + return None; + } + // Bytes por MCU depois da emenda, comparados com os do trecho anterior. + let after = d.dc.len() as f64 - (seam.saturating_sub(snap.mcu)) as f64; + let used = d.state.pos.saturating_sub(prefix.len()) as f64; + let ratio = if after > 4.0 && used > 0.0 { (used / after) / bpm_prefix.max(0.5) } else { 1.0 }; + let sc = sum / n as f64 + 20.0 * ratio.ln().abs(); + Some(sc) +} diff --git a/crates/guara-core/src/fs/ext.rs b/crates/guara-core/src/fs/ext.rs index 668826e..458dfc8 100644 --- a/crates/guara-core/src/fs/ext.rs +++ b/crates/guara-core/src/fs/ext.rs @@ -442,7 +442,7 @@ fn walk_dirs(ext: &Ext, ctx: &Ctx) -> Names { let mut seen: HashSet = HashSet::new(); let mut visited = 0usize; while let Some((dino, path)) = queue.pop() { - if !seen.insert(dino) || (ctx.cancelled)() || visited > 2_000_000 { + if !seen.insert(dino) || (visited % 256 == 255 && (ctx.cancelled)()) || visited > 2_000_000 { continue; } visited += 1; @@ -526,7 +526,7 @@ fn journal_inodes(ext: &Ext, ctx: &Ctx) -> HashMap>> { let mut idx = first; // Varredura física: qualquer bloco que seja descritor, em ordem do diário. while idx < n { - if (ctx.cancelled)() { + if idx % 4096 == 4095 && (ctx.cancelled)() { break; } let Some(d) = ext.read_block(jblocks[idx]) else { @@ -605,7 +605,7 @@ pub fn scan(ctx: &mut Ctx, base: u64, part_len: u64) -> VolumeScan { // 1) Inodes apagados na tabela atual. let total = ext.groups.len() as u64; for (g, gr) in ext.groups.iter().enumerate() { - if (ctx.cancelled)() { + if g % 16 == 15 && (ctx.cancelled)() { return res; } if gr.flags & 0x1 != 0 { @@ -660,9 +660,6 @@ pub fn scan(ctx: &mut Ctx, base: u64, part_len: u64) -> VolumeScan { if emitted.contains(ino_no) || names.live.contains_key(ino_no) || *ino_no < 11 { continue; } - if (ctx.cancelled)() { - return res; - } let cur = ext.read_inode(*ino_no); // Se o inode atual está vivo e é outro arquivo, não mexe. if cur.as_ref().map(|c| c.dtime == 0 && c.links > 0 && c.mode != 0).unwrap_or(false) { diff --git a/crates/guara-core/src/jpegscan.rs b/crates/guara-core/src/jpegscan.rs new file mode 100644 index 0000000..cbb3ff8 --- /dev/null +++ b/crates/guara-core/src/jpegscan.rs @@ -0,0 +1,598 @@ +//! Decodificador de entropia de JPEG sequencial (Huffman, 8 bits), sem IDCT. +//! +//! Serve para validar se os bytes de um JPEG são realmente contínuos e para reconstruir arquivos +//! fragmentados: o fluxo é decodificado MCU a MCU, guardando o DC de luminância (uma miniatura +//! 1/8 da imagem) e pontos de retomada. Depois de uma quebra os bytes viram lixo, e a +//! descontinuidade do DC (ou um erro de código) denuncia onde. + +#[derive(Clone, Debug)] +struct Huff { + // decodificação canônica + mincode: [i32; 17], + maxcode: [i32; 18], + valptr: [i32; 17], + vals: Vec, +} + +impl Huff { + fn new(counts: &[u8; 16], vals: Vec) -> Huff { + let mut h = Huff { mincode: [0; 17], maxcode: [-1; 18], valptr: [0; 17], vals }; + let mut code = 0i32; + let mut k = 0i32; + for l in 1..=16 { + h.valptr[l] = k; + h.mincode[l] = code; + code += counts[l - 1] as i32; + k += counts[l - 1] as i32; + h.maxcode[l] = if counts[l - 1] > 0 { code - 1 } else { -1 }; + code <<= 1; + } + h.maxcode[17] = i32::MAX; + h + } +} + +#[derive(Clone, Debug)] +pub struct Comp { + pub h: u8, + pub v: u8, + pub tq: usize, + pub dc_tbl: usize, + pub ac_tbl: usize, +} + +#[derive(Clone, Debug)] +pub struct Header { + pub width: u32, + pub height: u32, + pub comps: Vec, + pub restart: u32, + pub mcus_x: u32, + pub mcus_y: u32, + /// Onde começam os dados entropy-coded (depois do SOS). + pub scan_start: usize, + /// Tabelas de quantização (ordem zigzag). + pub qt: [[u16; 64]; 4], + dc: [Option; 4], + ac: [Option; 4], +} + +impl Header { + pub fn total_mcus(&self) -> u32 { + self.mcus_x * self.mcus_y + } +} + +/// Lê os segmentos até o SOS. `None` se não for JPEG sequencial suportado. +pub fn parse_header(d: &[u8]) -> Option
{ + if d.len() < 4 || d[0] != 0xFF || d[1] != 0xD8 { + return None; + } + let mut p = 2; + let mut dc: [Option; 4] = Default::default(); + let mut ac: [Option; 4] = Default::default(); + let mut sof: Option<(u32, u32, Vec<(u8, u8, usize)>)> = None; + let mut qt = [[1u16; 64]; 4]; + let mut restart = 0u32; + while p + 4 <= d.len() { + if d[p] != 0xFF { + return None; + } + let m = d[p + 1]; + if m == 0xFF { + p += 1; + continue; + } + if (0xD0..=0xD8).contains(&m) || m == 0x01 { + p += 2; + continue; + } + let len = u16::from_be_bytes([d[p + 2], d[p + 3]]) as usize; + if len < 2 || p + 2 + len > d.len() { + return None; + } + let seg = &d[p + 4..p + 2 + len]; + match m { + 0xC0 | 0xC1 => { + if seg.len() < 6 || seg[0] != 8 { + return None; + } + let h = u16::from_be_bytes([seg[1], seg[2]]) as u32; + let w = u16::from_be_bytes([seg[3], seg[4]]) as u32; + let n = seg[5] as usize; + if n == 0 || n > 4 || seg.len() < 6 + n * 3 || w == 0 || h == 0 { + return None; + } + let comps = (0..n).map(|i| (seg[7 + i * 3] >> 4, seg[7 + i * 3] & 15, seg[8 + i * 3] as usize & 3)).collect(); + sof = Some((w, h, comps)); + } + 0xC2 | 0xC3 | 0xC5..=0xC7 | 0xC9..=0xCB | 0xCD..=0xCF => return None, // progressivo, aritmético etc. + 0xC4 => { + let mut q = 0; + while q + 17 <= seg.len() { + let (cls, id) = (seg[q] >> 4, (seg[q] & 15) as usize); + let mut counts = [0u8; 16]; + counts.copy_from_slice(&seg[q + 1..q + 17]); + let n: usize = counts.iter().map(|&c| c as usize).sum(); + if id > 3 || q + 17 + n > seg.len() || n > 256 { + return None; + } + let h = Huff::new(&counts, seg[q + 17..q + 17 + n].to_vec()); + if cls == 0 { + dc[id] = Some(h); + } else { + ac[id] = Some(h); + } + q += 17 + n; + } + } + 0xDB => { + let mut q = 0; + while q < seg.len() { + let (prec, id) = (seg[q] >> 4, (seg[q] & 15) as usize); + q += 1; + let n = if prec == 0 { 64 } else { 128 }; + if id > 3 || q + n > seg.len() { + return None; + } + for i in 0..64 { + qt[id][i] = if prec == 0 { seg[q + i] as u16 } else { u16::from_be_bytes([seg[q + 2 * i], seg[q + 2 * i + 1]]) }; + } + q += n; + } + } + 0xDD => { + if seg.len() >= 2 { + restart = u16::from_be_bytes([seg[0], seg[1]]) as u32; + } + } + 0xDA => { + let (w, h, comps) = sof?; + let ns = *seg.first()? as usize; + if ns != comps.len() || seg.len() < 1 + ns * 2 + 3 { + return None; // varredura não intercalada + } + let mut cs = vec![]; + for i in 0..ns { + let t = seg[2 + i * 2]; + cs.push(Comp { h: comps[i].0, v: comps[i].1, tq: comps[i].2, dc_tbl: (t >> 4) as usize & 3, ac_tbl: (t & 15) as usize & 3 }); + } + let hmax = cs.iter().map(|c| c.h).max()? as u32; + let vmax = cs.iter().map(|c| c.v).max()? as u32; + if hmax == 0 || vmax == 0 || hmax > 4 || vmax > 4 { + return None; + } + for c in &cs { + if c.h == 0 || c.v == 0 || dc[c.dc_tbl].is_none() || ac[c.ac_tbl].is_none() { + return None; + } + } + return Some(Header { + width: w, + height: h, + comps: cs, + restart, + mcus_x: w.div_ceil(8 * hmax), + mcus_y: h.div_ceil(8 * vmax), + scan_start: p + 2 + len, + qt, + dc, + ac, + }); + } + _ => {} + } + p += 2 + len; + } + None +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Stop { + /// Todos os MCUs foram decodificados. + Done, + /// Marcador inesperado (inclusive EOI) na posição dada. + Marker(u8, usize), + /// Código inválido ou coeficiente fora do limite. + Error(usize), + /// Faltam bytes. + Truncated, +} + +/// Estado completo do decodificador num limite de byte, para retomar com outros dados. +#[derive(Clone, Debug)] +pub struct State { + /// Próximo byte a buscar no fluxo. + pub pos: usize, + bitbuf: u32, + bitcnt: u32, + pred: [i32; 4], + pub mcu: u32, + rst_next: u8, + mcus_since_rst: u32, +} + +impl State { + /// Último DC de luminância (preditor) no ponto de retomada. + pub fn luma_pred(&self) -> i32 { + self.pred[0] + } + pub fn start(h: &Header) -> State { + State { pos: h.scan_start, bitbuf: 0, bitcnt: 0, pred: [0; 4], mcu: 0, rst_next: 0, mcus_since_rst: 0 } + } +} + +pub struct Decoded { + /// DC de luminância (absoluto) por MCU decodificado. + pub dc: Vec, + /// Estados guardados em intervalos (para retomada): (MCU, estado ao fim dele). + pub snaps: Vec, + pub stop: Stop, + pub state: State, +} + +struct Bits<'a> { + d: &'a [u8], + st: State, + marker: Option<(u8, usize)>, +} + +impl Bits<'_> { + fn fill(&mut self) { + while self.st.bitcnt <= 24 && self.marker.is_none() { + let Some(&b) = self.d.get(self.st.pos) else { break }; + if b == 0xFF { + match self.d.get(self.st.pos + 1) { + Some(0) => { + self.st.bitbuf |= 0xFF << (24 - self.st.bitcnt); + self.st.bitcnt += 8; + self.st.pos += 2; + } + Some(&m) => { + self.marker = Some((m, self.st.pos)); + } + None => break, + } + } else { + self.st.bitbuf |= (b as u32) << (24 - self.st.bitcnt); + self.st.bitcnt += 8; + self.st.pos += 1; + } + } + } + fn get(&mut self, n: u32) -> Option { + if n == 0 { + return Some(0); + } + if self.st.bitcnt < n { + self.fill(); + if self.st.bitcnt < n { + return None; + } + } + let v = self.st.bitbuf >> (32 - n); + self.st.bitbuf <<= n; + self.st.bitcnt -= n; + Some(v) + } + fn huff(&mut self, h: &Huff) -> Option { + self.fill(); + let mut code = 0i32; + for l in 1..=16u32 { + if self.st.bitcnt < l { + return None; + } + code = (self.st.bitbuf >> (32 - l)) as i32; + if h.maxcode[l as usize] >= 0 && code <= h.maxcode[l as usize] && code >= h.mincode[l as usize] { + self.st.bitbuf <<= l; + self.st.bitcnt -= l; + let idx = h.valptr[l as usize] + code - h.mincode[l as usize]; + return h.vals.get(idx as usize).copied(); + } + } + let _ = code; + None + } +} + +fn extend(v: u32, t: u32) -> i32 { + if t == 0 { + 0 + } else if v < (1 << (t - 1)) { + v as i32 - (1 << t) + 1 + } else { + v as i32 + } +} + +/// Decodifica até `max_mcus` MCUs a partir de `state`, sobre `data` (fluxo bruto com FF00). +/// `snap_every` > 0 guarda estados a cada tantos bytes consumidos. +pub fn decode(h: &Header, data: &[u8], state: State, max_mcus: u32, snap_every: usize) -> Decoded { + decode_hook(h, data, state, max_mcus, snap_every, None) +} + +/// Como `decode`, mas entrega os coeficientes (zigzag) dos blocos de luminância de cada MCU. +pub fn decode_hook( + h: &Header, + data: &[u8], + state: State, + max_mcus: u32, + snap_every: usize, + mut hook: Option<&mut dyn FnMut(u32, &[[i32; 64]])>, +) -> Decoded { + let mut lblocks: Vec<[i32; 64]> = Vec::new(); + let mut b = Bits { d: data, st: state, marker: None }; + let total = h.total_mcus(); + let mut dc_out = Vec::new(); + let mut snaps = Vec::new(); + let mut last_snap = b.st.pos; + let mut stop = Stop::Done; + let mut decoded = 0u32; + 'mcu: while b.st.mcu < total && decoded < max_mcus { + // Marcador de reinício entre MCUs. + if h.restart > 0 && b.st.mcus_since_rst == h.restart { + // descarta bits de preenchimento e espera RSTn + b.st.bitbuf = 0; + b.st.bitcnt = 0; + let p = b.st.pos; + match (data.get(p), data.get(p + 1)) { + (Some(0xFF), Some(&m)) if m == 0xD0 + b.st.rst_next => { + b.st.pos += 2; + b.st.rst_next = (b.st.rst_next + 1) & 7; + b.st.mcus_since_rst = 0; + b.st.pred = [0; 4]; + b.marker = None; + } + (Some(0xFF), Some(&m)) => { + stop = Stop::Marker(m, p); + break 'mcu; + } + (None, _) | (_, None) => { + stop = Stop::Truncated; + break 'mcu; + } + _ => { + stop = Stop::Error(p); + break 'mcu; + } + } + } + let mut luma_dc = 0; + lblocks.clear(); + for (ci, c) in h.comps.iter().enumerate() { + let dct = h.dc[c.dc_tbl].as_ref().unwrap(); + let act = h.ac[c.ac_tbl].as_ref().unwrap(); + for blk in 0..(c.h as u32 * c.v as u32) { + let Some(t) = b.huff(dct) else { + stop = fail(&b, data); + break 'mcu; + }; + if t > 11 { + stop = Stop::Error(b.st.pos); + break 'mcu; + } + let Some(bits) = b.get(t as u32) else { + stop = fail(&b, data); + break 'mcu; + }; + b.st.pred[ci] += extend(bits, t as u32); + if ci == 0 && blk == 0 { + luma_dc = b.st.pred[0]; + } + let mut coef = [0i32; 64]; + if ci == 0 { + coef[0] = b.st.pred[0]; + } + let mut k = 1; + while k < 64 { + let Some(rs) = b.huff(act) else { + stop = fail(&b, data); + break 'mcu; + }; + let (r, s) = ((rs >> 4) as usize, (rs & 15) as u32); + if s == 0 { + if r == 15 { + k += 16; + continue; + } + break; + } + k += r; + if k > 63 { + stop = Stop::Error(b.st.pos); + break 'mcu; + } + let Some(vbits) = b.get(s) else { + stop = fail(&b, data); + break 'mcu; + }; + if ci == 0 && hook.is_some() { + coef[k] = extend(vbits, s); + } + k += 1; + } + if ci == 0 && hook.is_some() { + lblocks.push(coef); + } + } + } + if let Some(hk) = hook.as_mut() { + hk(b.st.mcu, &lblocks); + } + dc_out.push(luma_dc); + b.st.mcu += 1; + b.st.mcus_since_rst += 1; + decoded += 1; + if snap_every > 0 && b.st.pos >= last_snap + snap_every { + snaps.push(b.st.clone()); + last_snap = b.st.pos; + } + } + Decoded { dc: dc_out, snaps, stop, state: b.st } +} + +fn fail(b: &Bits, data: &[u8]) -> Stop { + if let Some((m, p)) = b.marker { + Stop::Marker(m, p) + } else if b.st.pos >= data.len() { + Stop::Truncated + } else { + Stop::Error(b.st.pos) + } +} + +/// Primeiro MCU suspeito: janela de `win` MCUs cuja diferença média de DC é anormalmente alta. +/// Devolve o índice do MCU onde a janela começa. +pub fn first_discontinuity(h: &Header, dc: &[i32]) -> Option { + let mx = h.mcus_x as usize; + let d: Vec = (0..dc.len()) + .map(|i| { + if i % mx > 0 { + (dc[i] - dc[i - 1]).abs() as f64 + } else if i >= mx { + (dc[i] - dc[i - mx]).abs() as f64 + } else { + 0.0 + } + }) + .collect(); + let win = 24usize; + if d.len() < win * 2 { + return None; + } + // Referência: mediana das diferenças vistas até aqui (histograma, robusta a bordas). + let mut hist = vec![0u32; 2048]; + let mut seen = 0u32; + let mut sum: f64 = d[..win].iter().sum(); + for x in &d[..win] { + hist[(*x as usize).min(2047)] += 1; + seen += 1; + } + for i in win..d.len() { + sum += d[i] - d[i - win]; + if i >= 64 + win && i % 8 == 0 { + let mut acc = 0u32; + let mut med = 0usize; + for (v, &c) in hist.iter().enumerate() { + acc += c; + if acc * 2 >= seen { + med = v; + break; + } + } + if sum / win as f64 > (6.0 * med as f64).max(40.0) { + return Some(i + 1 - win); + } + } + // Só entra na referência o que ainda parece normal. + if i < win * 3 || d[i] < 300.0 { + hist[(d[i] as usize).min(2047)] += 1; + seen += 1; + } + } + None +} + +// --------------------------------------------------------------- pixels + +const ZIGZAG: [usize; 64] = [ + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, + 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63, +]; + +fn cos_table() -> &'static [[f32; 8]; 8] { + static T: std::sync::OnceLock<[[f32; 8]; 8]> = std::sync::OnceLock::new(); + T.get_or_init(|| { + let mut t = [[0f32; 8]; 8]; + for x in 0..8 { + for u in 0..8 { + let cu = if u == 0 { std::f32::consts::FRAC_1_SQRT_2 } else { 1.0 }; + t[x][u] = 0.5 * cu * (((2 * x + 1) as f32 * u as f32 * std::f32::consts::PI) / 16.0).cos(); + } + } + t + }) +} + +/// IDCT de um bloco (coeficientes em zigzag, quantizados) para 8x8 pixels de luminância. +pub fn idct_block(coef: &[i32; 64], qt: &[u16; 64]) -> [u8; 64] { + let mut f = [[0f32; 8]; 8]; + for z in 0..64 { + let n = ZIGZAG[z]; + f[n / 8][n % 8] = (coef[z] * qt[z] as i32) as f32; + } + let t = cos_table(); + // linhas + let mut tmp = [[0f32; 8]; 8]; + for v in 0..8 { + for x in 0..8 { + let mut s = 0.0; + for u in 0..8 { + s += t[x][u] * f[v][u]; + } + tmp[v][x] = s; + } + } + let mut out = [0u8; 64]; + for y in 0..8 { + for x in 0..8 { + let mut s = 0.0; + for v in 0..8 { + s += t[y][v] * tmp[v][x]; + } + out[y * 8 + x] = (s + 128.0).round().clamp(0.0, 255.0) as u8; + } + } + out +} + +/// Luminância de um MCU (blocos `h` x `v` da primeira componente) como imagem (8h x 8v). +pub fn mcu_luma(h: &Header, blocks: &[[i32; 64]]) -> Vec { + let c = &h.comps[0]; + let (bw, bh) = (c.h as usize, c.v as usize); + let w = bw * 8; + let mut out = vec![0u8; w * bh * 8]; + for (i, blk) in blocks.iter().enumerate().take(bw * bh) { + let px = idct_block(blk, &h.qt[c.tq]); + let (bx, by) = (i % bw, i / bw); + for y in 0..8 { + out[(by * 8 + y) * w + bx * 8..(by * 8 + y) * w + bx * 8 + 8].copy_from_slice(&px[y * 8..y * 8 + 8]); + } + } + out +} + +/// Plano de luminância do que já foi decodificado. +pub struct Plane { + pub w: usize, + pub h: usize, + pub px: Vec, + pub mw: usize, + pub mh: usize, + pub have: u32, +} + +impl Plane { + pub fn new(h: &Header) -> Plane { + let c = &h.comps[0]; + let (mw, mh) = (c.h as usize * 8, c.v as usize * 8); + Plane { w: mw * h.mcus_x as usize, h: mh * h.mcus_y as usize, px: vec![0; mw * h.mcus_x as usize * mh * h.mcus_y as usize], mw, mh, have: 0 } + } + pub fn put(&mut self, h: &Header, mcu: u32, luma: &[u8]) { + let (mx, my) = ((mcu % h.mcus_x) as usize, (mcu / h.mcus_x) as usize); + for y in 0..self.mh { + let dst = (my * self.mh + y) * self.w + mx * self.mw; + self.px[dst..dst + self.mw].copy_from_slice(&luma[y * self.mw..(y + 1) * self.mw]); + } + self.have = self.have.max(mcu + 1); + } +} + +/// Decodifica e preenche o plano com as luminâncias. +pub fn decode_plane(h: &Header, data: &[u8], state: State, max_mcus: u32, plane: &mut Plane) -> Decoded { + let mut hook = |mcu: u32, blocks: &[[i32; 64]]| { + let l = mcu_luma(h, blocks); + plane.put(h, mcu, &l); + }; + decode_hook(h, data, state, max_mcus, 512, Some(&mut hook)) +} diff --git a/crates/guara-core/src/lib.rs b/crates/guara-core/src/lib.rs index d11e128..cbce27e 100644 --- a/crates/guara-core/src/lib.rs +++ b/crates/guara-core/src/lib.rs @@ -5,6 +5,7 @@ pub mod assemble; pub mod carve; +pub mod defrag; pub mod device; pub mod crypt; pub mod error; @@ -16,6 +17,7 @@ pub mod hashing; pub mod health; pub mod http; pub mod image; +pub mod jpegscan; pub mod lvm; pub mod meta; pub mod notify; diff --git a/crates/guara-core/src/scan.rs b/crates/guara-core/src/scan.rs index 5ce4376..e14050d 100644 --- a/crates/guara-core/src/scan.rs +++ b/crates/guara-core/src/scan.rs @@ -23,6 +23,10 @@ pub trait Sink { fn found(&mut self, f: Found); /// Um achado de carving confirmou um arquivo já listado por metadados. fn confirm(&mut self, id: u64, confidence: Confidence); + /// Um achado já emitido ganhou dados melhores (ex.: JPEG remontado de fragmentos). + fn replace(&mut self, f: Found) { + self.found(f); + } fn progress(&mut self, p: Progress); fn warning(&mut self, w: String); fn layout(&mut self, l: &DiskLayout); @@ -194,12 +198,16 @@ pub fn run( categories: &opts.categories, }; let mut handler = Handler { + src, sink, starts, fs_extents, next_id: &next_id, done, total, + index: crate::defrag::ClusterIndex::new(src.size()), + pending: vec![], + carved: vec![], }; for ri in 0..cp.ranges.len() { let (start, end) = (cp.ranges[ri].start, cp.ranges[ri].end); @@ -215,19 +223,58 @@ pub fn run( return Outcome::Interrupted(cp); } } + handler.rebuild_pending(); Outcome::Completed } +/// JPEG que parece quebrado: tenta recompor depois de varrer o disco todo. +struct PendingJpeg { + found: Found, + hit_len: u64, +} + struct Handler<'a> { + src: &'a dyn Source, sink: &'a mut dyn Sink, starts: HashMap, fs_extents: Vec, next_id: &'a AtomicU64, done: u64, total: u64, + index: crate::defrag::ClusterIndex, + pending: Vec, + /// Extents de arquivos carved já emitidos (não podem servir de fragmento para outro). + carved: Vec, +} + +impl Handler<'_> { + fn rebuild_pending(&mut self) { + let pend = std::mem::take(&mut self.pending); + for p in pend { + let off = p.found.first_offset().unwrap_or(0); + let mut used = self.fs_extents.clone(); + used.extend(self.carved.iter().copied()); + // O próprio começo do arquivo não conta como "ocupado" para ele mesmo. + used.retain(|e| !(e.offset == off)); + let pool = crate::defrag::Pool { index: &self.index, used: &used }; + if let crate::defrag::Analysis::Rebuilt { extents, size } = crate::defrag::analyze_jpeg(self.src, off, p.hit_len, &pool, true) { + let mut f = p.found; + f.size = size; + f.data = DataLoc::Extents { extents: extents.clone() }; + f.confidence = Confidence::Medium; + f.note = Some(format!("remontado a partir de {} fragmentos separados no disco; confira a imagem", extents.len())); + self.carved.extend(extents); + self.sink.replace(f); + } + } + } } impl CarveEvents for Handler<'_> { + fn chunk(&mut self, abs: u64, data: &[u8]) { + self.index.scan_chunk(abs, data); + } + fn advance(&mut self, n: u64) { self.done += n; self.sink.progress(Progress { @@ -248,6 +295,26 @@ impl CarveEvents for Handler<'_> { if overlaps(&self.fs_extents, off, 1) { return HitAction::Continue; // dentro de arquivo já listado (ex.: miniatura embutida) } + let mut hit = hit.clone(); + let mut skip_len = None; + let mut pending_len = None; + if hit.ext == "jpg" { + let pool = crate::defrag::Pool { index: &self.index, used: &[] }; + match crate::defrag::analyze_jpeg(self.src, off, hit.len, &pool, false) { + crate::defrag::Analysis::Intact { len } => { + hit.len = len; + hit.confidence = Confidence::High; + } + crate::defrag::Analysis::Broken { good_len } => { + pending_len = Some(hit.len); + hit.len = good_len; + hit.confidence = Confidence::Low; + skip_len = Some(good_len); + } + crate::defrag::Analysis::Unsupported | crate::defrag::Analysis::Rebuilt { .. } => {} + } + } + let hit = &hit; let id = self .next_id .fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -278,8 +345,17 @@ impl CarveEvents for Handler<'_> { }, }; self.starts.insert(off, (id, hit.confidence)); + self.carved.push(Extent::new(off, hit.len)); + if let Some(hl) = pending_len { + let mut pf = f.clone(); + pf.note = Some("JPEG interrompido ou fragmentado".into()); + self.pending.push(PendingJpeg { found: pf, hit_len: hl }); + } self.sink.found(f); - HitAction::Skip + match skip_len { + Some(n) => HitAction::SkipLen(n), + None => HitAction::Skip, + } } } diff --git a/crates/guara-core/src/service.rs b/crates/guara-core/src/service.rs index f50fcb2..e352e86 100644 --- a/crates/guara-core/src/service.rs +++ b/crates/guara-core/src/service.rs @@ -137,6 +137,17 @@ impl Sink for JobSink { s.found[i].confidence = s.found[i].confidence.max(c); } } + fn replace(&mut self, f: Found) { + let mut s = self.0.st.lock().unwrap(); + match s.index.get(&f.id).copied() { + Some(i) => s.found[i] = f, + None => { + let i = s.found.len(); + s.index.insert(f.id, i); + s.found.push(f); + } + } + } fn progress(&mut self, p: Progress) { self.0.st.lock().unwrap().progress = p; } @@ -738,17 +749,6 @@ impl Engine { } pub fn load_session(&self, path: &str) -> Result { - // Clicar de novo numa sessão já aberta reaproveita o job em vez de criar outro igual. - let want = Path::new(path); - if let Some(j) = self - .jobs - .lock() - .unwrap() - .values() - .find(|j| j.session_file.lock().unwrap().as_deref() == Some(want)) - { - return Ok(j.id.clone()); - } let bytes = std::fs::read(path).map_err(|e| { if e.kind() == std::io::ErrorKind::PermissionDenied { Error::msg(format!( @@ -763,6 +763,17 @@ impl Engine { "O arquivo {path} não é uma sessão do Guará válida (ou foi criado por uma versão incompatível): {e}" )) })?; + // Clicar de novo numa sessão já aberta reaproveita o job em vez de criar outro igual. + let want = Path::new(path); + if let Some(j) = self + .jobs + .lock() + .unwrap() + .values() + .find(|j| j.session_file.lock().unwrap().as_deref() == Some(want)) + { + return Ok(j.id.clone()); + } let job = self.register( JobKind::Scan, data.target.clone(), diff --git a/crates/guara-core/tests/common/mod.rs b/crates/guara-core/tests/common/mod.rs index 823dc33..223473c 100644 --- a/crates/guara-core/tests/common/mod.rs +++ b/crates/guara-core/tests/common/mod.rs @@ -22,6 +22,12 @@ impl Sink for Collect { f.confidence = f.confidence.max(c); } } + fn replace(&mut self, f: Found) { + match self.found.iter_mut().find(|x| x.id == f.id) { + Some(x) => *x = f, + None => self.found.push(f), + } + } fn progress(&mut self, _: Progress) {} fn warning(&mut self, w: String) { self.warnings.push(w); diff --git a/crates/guara-core/tests/defrag.rs b/crates/guara-core/tests/defrag.rs new file mode 100644 index 0000000..f686b98 --- /dev/null +++ b/crates/guara-core/tests/defrag.rs @@ -0,0 +1,183 @@ +use guara_core::defrag::*; +use guara_core::source::MemSource; +use guara_core::types::Extent; + +fn photo(w: u16, h: u16, seed: u64, shift: i32) -> Vec { + photo_n(w, h, seed, shift, 31) +} + +fn photo_n(w: u16, h: u16, seed: u64, shift: i32, nmask: u64) -> Vec { + let mut rgb = Vec::with_capacity(w as usize * h as usize * 3); + let mut x = seed | 1; + for yy in 0..h as i32 { + for xx in 0..w as i32 { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + let noise = ((x >> 40) & nmask) as i32; + let fx = xx as f32 / w as f32; + let fy = yy as f32 / h as f32; + let wave = ((fx * 9.0 + seed as f32 * 0.37).sin() * (fy * 7.0 + 1.3).cos() * 40.0) as i32 + ((fx * 31.0).sin() * 8.0) as i32; + let base = 110 + shift + wave + (xx * 40 / w as i32); + let blob = if ((xx - 300).pow(2) + (yy - 200).pow(2)) < 5000 { 60 } else { 0 }; + let v = (base + blob + noise).clamp(0, 255) as u8; + rgb.extend_from_slice(&[v, v.saturating_sub(10), (v as i32 / 2 + noise) as u8]); + } + } + let mut out = vec![]; + jpeg_encoder::Encoder::new(&mut out, 92).encode(&rgb, w, h, jpeg_encoder::ColorType::Rgb).unwrap(); + out +} + +fn noise(n: usize, seed: u64) -> Vec { + let mut x = seed | 1; + (0..n) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 24) as u8 + }) + .collect() +} + +fn pad(v: &mut Vec) { + v.resize(v.len().div_ceil(4096) * 4096, 0); +} + +#[test] +fn rebuilds_a_jpeg_split_around_another_one() { + let a = photo(1024, 768, 7, 0); + let b = photo(800, 600, 99, 40); + let split = 14 * 4096; + assert!(a.len() > split + 3 * 4096, "A pequena: {}", a.len()); + let mut disk: Vec = vec![0u8; 64 * 4096]; + let a_off = disk.len() as u64; + disk.extend_from_slice(&a[..split]); + let b_off = disk.len() as u64; + let mut bb = b.clone(); + pad(&mut bb); + disk.extend_from_slice(&bb); + disk.extend(noise(20 * 4096, 5)); // lixo + let a2_off = disk.len() as u64; + let mut rest = a[split..].to_vec(); + pad(&mut rest); + disk.extend_from_slice(&rest); + disk.extend(noise(10 * 4096, 6)); + let src = MemSource(disk.clone()); + + let mut idx = ClusterIndex::new(disk.len() as u64); + idx.scan_chunk(0, &disk); + // O fragmento correto precisa estar marcado. + assert!(idx.get(a2_off)); + eprintln!("TRUE a2_off={a2_off}"); + let pool = Pool { index: &idx, used: &[Extent::new(b_off, bb.len() as u64)] }; + match analyze_jpeg(&src, a_off, 4 << 20, &pool, true) { + Analysis::Rebuilt { extents, size } => { + assert_eq!(size as usize, a.len(), "tamanho"); + assert_eq!(extents.len(), 2); + assert_eq!(extents[0], Extent::new(a_off, split as u64)); + assert_eq!(extents[1].offset, a2_off); + let mut got = disk[a_off as usize..a_off as usize + split].to_vec(); + got.extend_from_slice(&disk[a2_off as usize..a2_off as usize + (size as usize - split)]); + assert!(got == a, "conteúdo recomposto difere"); + } + other => panic!("esperava Rebuilt, veio {other:?}"), + } + // O outro JPEG, íntegro, continua íntegro. + let pool2 = Pool { index: &idx, used: &[] }; + match analyze_jpeg(&src, b_off, 4 << 20, &pool2, true) { + Analysis::Intact { len } => assert_eq!(len as usize, b.len()), + other => panic!("B deveria estar íntegro: {other:?}"), + } +} + +#[test] +fn rebuilds_three_fragments_among_many_decoys() { + let a = photo_n(2048, 1536, 21, 10, 3); + let n = a.len(); + let (s1, s2) = (10 * 4096, 10 * 4096 + (n - 10 * 4096) / 2 / 4096 * 4096); + assert!(n > s2 + 4 * 4096); + let mut disk: Vec = vec![0u8; 32 * 4096]; + let a_off = disk.len() as u64; + disk.extend_from_slice(&a[..s1]); + let mut decoys = vec![]; + for k in 0..6u64 { + let mut d = photo_n(1600, 1200, 100 + k, (k as i32) * 17, 3); + pad(&mut d); + decoys.push((disk.len() as u64, d.len() as u64)); + disk.extend_from_slice(&d); + if k == 1 { + // segundo fragmento de A no meio + let off2 = disk.len() as u64; + disk.extend_from_slice(&a[s1..s2]); + decoys.push((off2, (s2 - s1) as u64)); + } + if k == 4 { + let mut r = a[s2..].to_vec(); + pad(&mut r); + disk.extend_from_slice(&r); + } + } + let src = MemSource(disk.clone()); + let mut idx = ClusterIndex::new(disk.len() as u64); + idx.scan_chunk(0, &disk); + eprintln!("TRUE frag2={:?}", decoys[2]); + let used: Vec = decoys + .iter() + .enumerate() + .filter(|(i, _)| *i != 2) // o fragmento do meio de A não é de outro arquivo + .map(|(_, (o, l))| Extent::new(*o, *l)) + .collect(); + let pool = Pool { index: &idx, used: &used }; + match analyze_jpeg(&src, a_off, 4 << 20, &pool, true) { + Analysis::Rebuilt { extents, size } => { + assert_eq!(size as usize, n); + let mut got = vec![]; + for e in &extents { + got.extend_from_slice(&disk[e.offset as usize..(e.offset + e.len) as usize]); + } + assert!(got == a, "conteúdo difere ({} fragmentos)", extents.len()); + } + other => panic!("{other:?}"), + } +} + +mod common; + +#[test] +fn deep_scan_recovers_fragmented_jpeg_end_to_end() { + use guara_core::{recover, ScanMode, ScanOptions}; + common::init(); + let a = photo_n(2048, 1536, 5, 0, 3); + let n = a.len(); + let split = 12 * 4096; + let mut disk: Vec = vec![0u8; 16 * 4096]; + let a_off = disk.len(); + disk.extend_from_slice(&a[..split]); + let mut b = photo_n(1600, 1200, 77, 30, 3); + pad(&mut b); + let b_len = b.len(); + disk.extend_from_slice(&b); + disk.extend(noise(8 * 4096, 3)); + let mut rest = a[split..].to_vec(); + pad(&mut rest); + disk.extend_from_slice(&rest); + disk.extend(noise(8 * 4096, 4)); + let dir = tempfile::tempdir().unwrap(); + let img = dir.path().join("d.img"); + std::fs::write(&img, &disk).unwrap(); + let c = common::scan_path(&img, &ScanOptions { mode: ScanMode::Deep, ..Default::default() }); + let jpgs: Vec<_> = c.found.iter().filter(|f| f.ext == "jpg").collect(); + assert_eq!(jpgs.len(), 2, "{:?}", jpgs.iter().map(|f| (f.size, f.first_offset(), f.confidence)).collect::>()); + let fa = jpgs.iter().find(|f| f.first_offset() == Some(a_off as u64)).unwrap(); + assert_eq!(fa.size as usize, n, "{:?}", fa.note); + let out = dir.path().join("out"); + let rep = common::recover_all(&img, std::slice::from_ref(*fa), &out); + let p = std::path::PathBuf::from(&rep.recovered[0].path); + let got = std::fs::read(if p.is_absolute() { p } else { out.join(p) }).unwrap(); + assert!(got == a, "JPEG recuperado difere do original"); + let fb = jpgs.iter().find(|f| f.first_offset() != Some(a_off as u64)).unwrap(); + assert_eq!(fb.size as usize, photo_n(1600, 1200, 77, 30, 3).len()); + let _ = (b_len, recover::RecoverReport::default()); +} diff --git a/crates/guara-core/tests/jpegscan.rs b/crates/guara-core/tests/jpegscan.rs new file mode 100644 index 0000000..a5fbc5a --- /dev/null +++ b/crates/guara-core/tests/jpegscan.rs @@ -0,0 +1,70 @@ +use guara_core::jpegscan::*; + +fn photo(w: u16, h: u16) -> Vec { + // Imagem "natural": gradientes suaves + textura + bordas. + let mut rgb = Vec::with_capacity(w as usize * h as usize * 3); + let mut x = 0x9E3779B97F4A7C15u64; + for yy in 0..h as i32 { + for xx in 0..w as i32 { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + let noise = ((x >> 40) & 31) as i32; + let base = 90 + (xx * 90 / w as i32) + (yy * 60 / h as i32); + let blob = if ((xx - 200).pow(2) + (yy - 150).pow(2)) < 3600 { 70 } else { 0 }; + let v = (base + blob + noise).clamp(0, 255) as u8; + rgb.extend_from_slice(&[v, v.saturating_sub(20), (v as i32 / 2 + noise) as u8]); + } + } + let mut out = vec![]; + let enc = jpeg_encoder::Encoder::new(&mut out, 92); + enc.encode(&rgb, w, h, jpeg_encoder::ColorType::Rgb).unwrap(); + out +} + +#[test] +fn decodes_whole_image_and_ends_at_eoi() { + let j = photo(1024, 768); + let h = parse_header(&j).expect("cabeçalho"); + assert_eq!((h.width, h.height), (1024, 768)); + let d = decode(&h, &j, State::start(&h), u32::MAX, 0); + assert_eq!(d.stop, Stop::Done); + assert_eq!(d.dc.len() as u32, h.total_mcus()); + // O que sobra depois do último MCU é o EOI. + let tail = &j[d.state.pos.min(j.len())..]; + assert!(tail.windows(2).any(|w| w == [0xFF, 0xD9]), "sem EOI depois do último MCU"); + assert!(first_discontinuity(&h, &d.dc).is_none()); +} + +#[test] +fn detects_break_and_resumes_with_the_right_continuation() { + let j = photo(1024, 768); + let h = parse_header(&j).unwrap(); + let cut = 12 * 4096; // quebra no limite de cluster + assert!(j.len() > cut + 8192, "imagem pequena demais: {}", j.len()); + // Dados estranhos no lugar do resto. + let mut x = 12345u64; + let junk: Vec = (0..40_000) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 24) as u8 + }) + .collect(); + let mut broken = j[..cut].to_vec(); + broken.extend_from_slice(&junk); + let d = decode(&h, &broken, State::start(&h), u32::MAX, 512); + assert_ne!(d.stop, Stop::Done); + // Decodificação segue o lixo por um tempo; mas a quebra aparece pelo erro ou pela descontinuidade. + let mcu_break = first_discontinuity(&h, &d.dc); + // MCUs bons: os decodificados ao fim do cluster `cut`. + let good = d.snaps.iter().filter(|s| s.pos <= cut).last().expect("snapshot antes da quebra").clone(); + // Retomada com a continuação verdadeira reproduz a decodificação inteira. + let mut joined = j[..cut].to_vec(); + joined.extend_from_slice(&j[cut..]); + let r = decode(&h, &joined, good.clone(), u32::MAX, 0); + assert_eq!(r.stop, Stop::Done); + assert_eq!(good.mcu as usize + r.dc.len(), h.total_mcus() as usize); + let _ = mcu_break; +}