From df194edb2f363b0e02e3ea40c5aab5f63de72048 Mon Sep 17 00:00:00 2001 From: Lucas Rayan Date: Mon, 19 Oct 2026 00:06:25 -0300 Subject: [PATCH] =?UTF-8?q?Reconstru=C3=A7=C3=A3o=20de=20v=C3=ADdeo=20MP4/?= =?UTF-8?q?MOV=20sem=20moov=20(H.264/HEVC)=20com=20refer=C3=AAncia=20de=20?= =?UTF-8?q?c=C3=A2mera=20e=20arquivos=20montados=20em=20partes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/guara-core/Cargo.toml | 1 + crates/guara-core/src/fs/mod.rs | 1 + crates/guara-core/src/lib.rs | 1 + crates/guara-core/src/mp4rebuild.rs | 1112 +++++++++++++++++++++++++ crates/guara-core/src/recover.rs | 42 + crates/guara-core/src/scan.rs | 71 +- crates/guara-core/src/types.rs | 74 ++ crates/guara-core/tests/mp4rebuild.rs | 309 +++++++ 8 files changed, 1606 insertions(+), 5 deletions(-) create mode 100644 crates/guara-core/src/mp4rebuild.rs create mode 100644 crates/guara-core/tests/mp4rebuild.rs diff --git a/crates/guara-core/Cargo.toml b/crates/guara-core/Cargo.toml index dccdc29..bff50ee 100644 --- a/crates/guara-core/Cargo.toml +++ b/crates/guara-core/Cargo.toml @@ -34,3 +34,4 @@ windows-sys = { version = "0.59", features = ["Win32_Foundation", "Win32_Storage [dev-dependencies] tempfile = "3" +mp4 = "0.14" diff --git a/crates/guara-core/src/fs/mod.rs b/crates/guara-core/src/fs/mod.rs index 8bd3e8f..888b54f 100644 --- a/crates/guara-core/src/fs/mod.rs +++ b/crates/guara-core/src/fs/mod.rs @@ -66,6 +66,7 @@ pub fn make_found(ctx: &Ctx, n: NewFound) -> Found { let mut note = n.note.map(String::from); let head: Vec = match &n.data { DataLoc::Resident { bytes } => bytes.iter().take(32).copied().collect(), + DataLoc::Parts { .. } => vec![], DataLoc::Extents { extents } => extents .iter() .find(|e| !e.sparse) diff --git a/crates/guara-core/src/lib.rs b/crates/guara-core/src/lib.rs index cbce27e..7ddf2a2 100644 --- a/crates/guara-core/src/lib.rs +++ b/crates/guara-core/src/lib.rs @@ -20,6 +20,7 @@ pub mod image; pub mod jpegscan; pub mod lvm; pub mod meta; +pub mod mp4rebuild; pub mod notify; pub mod owner; pub mod partition; diff --git a/crates/guara-core/src/mp4rebuild.rs b/crates/guara-core/src/mp4rebuild.rs new file mode 100644 index 0000000..186ae15 --- /dev/null +++ b/crates/guara-core/src/mp4rebuild.rs @@ -0,0 +1,1112 @@ +//! Reconstrução de vídeo MP4/MOV sem o átomo `moov` (gravação interrompida de câmera ou drone, +//! arquivo cortado). Os quadros ficam no `mdat` como unidades NAL com prefixo de tamanho; o +//! índice (`moov`) é refeito a partir delas: tamanhos de amostra, quadros-chave, tempo e, para +//! H.264, a ordem de apresentação (POC) quando há quadros B. O áudio não é reconstruído. + +use crate::source::{read_vec, Source}; + +fn be32(b: &[u8], o: usize) -> u32 { + b.get(o..o + 4).map(|s| u32::from_be_bytes(s.try_into().unwrap())).unwrap_or(0) +} + +// ------------------------------------------------------------------ bits + +struct Bits<'a> { + d: &'a [u8], + pos: usize, +} + +impl<'a> Bits<'a> { + fn new(d: &'a [u8]) -> Self { + Self { d, pos: 0 } + } + fn bit(&mut self) -> Option { + let b = *self.d.get(self.pos / 8)?; + let v = (b >> (7 - self.pos % 8)) & 1; + self.pos += 1; + Some(v as u32) + } + fn u(&mut self, n: u32) -> Option { + let mut v = 0u32; + for _ in 0..n { + v = (v << 1) | self.bit()?; + } + Some(v) + } + fn ue(&mut self) -> Option { + let mut z = 0; + while self.bit()? == 0 { + z += 1; + if z > 31 { + return None; + } + } + Some(((1u64 << z) - 1 + self.u(z)? as u64) as u32) + } + fn se(&mut self) -> Option { + let k = self.ue()?; + Some(if k % 2 == 1 { (k / 2 + 1) as i32 } else { -((k / 2) as i32) }) + } + fn skip(&mut self, n: usize) { + self.pos += n; + } +} + +/// Remove os bytes de "emulation prevention" (00 00 03 -> 00 00). +fn unescape(nal: &[u8]) -> Vec { + let mut out = Vec::with_capacity(nal.len()); + let mut z = 0; + for &b in nal { + if z >= 2 && b == 3 { + z = 0; + continue; + } + z = if b == 0 { z + 1 } else { 0 }; + out.push(b); + } + out +} + +// ------------------------------------------------------------------ H.264 + +#[derive(Clone, Debug, Default)] +pub struct Sps264 { + pub profile: u8, + pub compat: u8, + pub level: u8, + pub width: u32, + pub height: u32, + pub log2_max_frame_num: u32, + pub poc_type: u32, + pub log2_max_poc_lsb: u32, + pub frame_mbs_only: bool, + /// (unidades por tique, escala de tempo) + pub timing: Option<(u32, u32)>, +} + +fn skip_scaling_list(b: &mut Bits, n: usize) -> Option<()> { + let (mut last, mut next) = (8i32, 8i32); + for _ in 0..n { + if next != 0 { + next = (last + b.se()? + 256) % 256; + } + last = if next == 0 { last } else { next }; + } + Some(()) +} + +pub fn parse_sps264(nal: &[u8]) -> Option { + if nal.len() < 5 || nal[0] & 0x1F != 7 { + return None; + } + let rbsp = unescape(&nal[1..]); + let mut b = Bits::new(&rbsp); + let mut s = Sps264 { profile: b.u(8)? as u8, compat: b.u(8)? as u8, level: b.u(8)? as u8, ..Default::default() }; + b.ue()?; // sps_id + let mut chroma = 1; + if matches!(s.profile, 100 | 110 | 122 | 244 | 44 | 83 | 86 | 118 | 128 | 138 | 139 | 134 | 135) { + chroma = b.ue()?; + if chroma == 3 { + b.bit()?; + } + b.ue()?; + b.ue()?; + b.bit()?; + if b.bit()? == 1 { + for i in 0..(if chroma != 3 { 8 } else { 12 }) { + if b.bit()? == 1 { + skip_scaling_list(&mut b, if i < 6 { 16 } else { 64 })?; + } + } + } + } + s.log2_max_frame_num = b.ue()? + 4; + s.poc_type = b.ue()?; + match s.poc_type { + 0 => s.log2_max_poc_lsb = b.ue()? + 4, + 1 => { + b.bit()?; + b.se()?; + b.se()?; + let n = b.ue()?; + for _ in 0..n.min(255) { + b.se()?; + } + } + _ => {} + } + b.ue()?; // max_num_ref_frames + b.bit()?; + let w = b.ue()? + 1; + let h = b.ue()? + 1; + s.frame_mbs_only = b.bit()? == 1; + if !s.frame_mbs_only { + b.bit()?; + } + b.bit()?; + let (mut cl, mut cr, mut ct, mut cb) = (0, 0, 0, 0); + if b.bit()? == 1 { + cl = b.ue()?; + cr = b.ue()?; + ct = b.ue()?; + cb = b.ue()?; + } + let (sub_w, sub_h) = match chroma { + 0 => (1, 2 - s.frame_mbs_only as u32), + 1 => (2, 2 * (2 - s.frame_mbs_only as u32)), + 2 => (2, 2 - s.frame_mbs_only as u32), + _ => (1, 2 - s.frame_mbs_only as u32), + }; + s.width = (w * 16).checked_sub((cl + cr) * sub_w)?; + s.height = ((2 - s.frame_mbs_only as u32) * h * 16).checked_sub((ct + cb) * sub_h)?; + // VUI até o timing_info. + if b.bit()? == 1 { + if b.bit()? == 1 { + if b.u(8)? == 255 { + b.skip(32); + } + } + if b.bit()? == 1 { + b.bit()?; + } + if b.bit()? == 1 { + b.skip(4); + if b.bit()? == 1 { + b.skip(24); + } + } + if b.bit()? == 1 { + b.ue()?; + b.ue()?; + } + if b.bit()? == 1 { + let units = b.u(32)?; + let scale = b.u(32)?; + if units > 0 && scale > 0 { + s.timing = Some((units, scale)); + } + } + } + Some(s) +} + +/// (primeiro_mb == 0, tipo de fatia, pps_id, frame_num, idr, poc_lsb) do cabeçalho de fatia H.264. +struct SliceInfo { + frame_num: u32, + first_mb0: bool, + idr: bool, + poc_lsb: Option, + bottom: bool, +} + +fn slice264(nal: &[u8], sps: &Sps264, pic_order_present: bool) -> Option { + let rbsp = unescape(&nal[1..nal.len().min(48)]); + let mut b = Bits::new(&rbsp); + let first = b.ue()?; + b.ue()?; // slice_type + b.ue()?; // pps_id + let frame_num = b.u(sps.log2_max_frame_num)?; + let mut bottom = false; + if !sps.frame_mbs_only { + if b.bit()? == 1 { + bottom = b.bit()? == 1; + } + } + let idr = nal[0] & 0x1F == 5; + if idr { + b.ue()?; + } + let mut poc_lsb = None; + if sps.poc_type == 0 { + poc_lsb = Some(b.u(sps.log2_max_poc_lsb)?); + let _ = pic_order_present; + } + Some(SliceInfo { frame_num, first_mb0: first == 0, idr, poc_lsb, bottom }) +} + +/// bottom_field_pic_order_in_frame_present_flag do PPS. +fn pps_pic_order_present(nal: &[u8]) -> bool { + let rbsp = unescape(&nal[1..]); + let mut b = Bits::new(&rbsp); + let (Some(_), Some(_), Some(_)) = (b.ue(), b.ue(), b.bit()) else { return false }; + b.bit() == Some(1) +} + +// ------------------------------------------------------------------ HEVC + +#[derive(Clone, Debug, Default)] +pub struct SpsHevc { + pub width: u32, + pub height: u32, + pub ptl: [u8; 12], + pub chroma: u32, + pub bit_depth_luma: u32, + pub bit_depth_chroma: u32, + pub timing: Option<(u32, u32)>, +} + +pub fn parse_sps_hevc(nal: &[u8]) -> Option { + if nal.len() < 8 || (nal[0] >> 1) & 0x3F != 33 { + return None; + } + let rbsp = unescape(&nal[2..]); + let mut b = Bits::new(&rbsp); + b.u(4)?; // vps id + let max_sub = b.u(3)? as usize; + b.bit()?; + // profile_tier_level + let mut s = SpsHevc::default(); + for (i, v) in s.ptl.iter_mut().enumerate() { + *v = b.u(8)? as u8; + let _ = i; + } + // general: 2+1+5 bits, 32 compat, 48 flags, 8 level => lemos 12 bytes (1+4+6+1) + let mut sub_profile = vec![]; + let mut sub_level = vec![]; + for _ in 0..max_sub { + sub_profile.push(b.bit()?); + sub_level.push(b.bit()?); + } + if max_sub > 0 { + for _ in max_sub..8 { + b.u(2)?; + } + } + for i in 0..max_sub { + if sub_profile[i] == 1 { + b.skip(88); + } + if sub_level[i] == 1 { + b.skip(8); + } + } + b.ue()?; // sps id + s.chroma = b.ue()?; + if s.chroma == 3 { + b.bit()?; + } + s.width = b.ue()?; + s.height = b.ue()?; + if b.bit()? == 1 { + let (l, r, t, bt) = (b.ue()?, b.ue()?, b.ue()?, b.ue()?); + let (sw, sh) = match s.chroma { + 1 => (2, 2), + 2 => (2, 1), + _ => (1, 1), + }; + s.width = s.width.checked_sub((l + r) * sw)?; + s.height = s.height.checked_sub((t + bt) * sh)?; + } + s.bit_depth_luma = b.ue()? + 8; + s.bit_depth_chroma = b.ue()? + 8; + Some(s) +} + +// -------------------------------------------------------------- amostras + +/// Configuração de codec (SPS/PPS/VPS) extraída de um vídeo íntegro da mesma câmera. +#[derive(Clone, Debug)] +pub struct MediaConfig { + pub codec: Codec, + pub sps: Vec>, + pub pps: Vec>, + pub vps: Vec>, +} + +/// Procura o `avcC`/`hvcC` de um MP4 completo (lê até `max` bytes a partir de `off`). +pub fn config_from_mp4(src: &dyn Source, off: u64, max: u64) -> Option { + // O moov costuma estar no começo ou no fim: procura nos dois lugares. + let size = src.size(); + let head = read_vec(src, off, max.min(4 << 20) as usize).ok()?; + let mut hay: Vec> = vec![head]; + if max > (4 << 20) && off + max <= size { + let tail_off = off + max - (4 << 20).min(max); + hay.push(read_vec(src, tail_off, (off + max - tail_off) as usize).ok()?); + } + for h in hay { + for i in 4..h.len().saturating_sub(12) { + let tag = &h[i..i + 4]; + if tag == b"avcC" { + let c = &h[i + 4..]; + if c.len() > 8 && c[0] == 1 { + let mut p = 6; + let mut sps = vec![]; + for _ in 0..(c[5] & 0x1F) { + let l = u16::from_be_bytes([*c.get(p)?, *c.get(p + 1)?]) as usize; + sps.push(c.get(p + 2..p + 2 + l)?.to_vec()); + p += 2 + l; + } + let np = *c.get(p)? as usize; + p += 1; + let mut pps = vec![]; + for _ in 0..np { + let l = u16::from_be_bytes([*c.get(p)?, *c.get(p + 1)?]) as usize; + pps.push(c.get(p + 2..p + 2 + l)?.to_vec()); + p += 2 + l; + } + if !sps.is_empty() && !pps.is_empty() { + return Some(MediaConfig { codec: Codec::Avc, sps, pps, vps: vec![] }); + } + } + } else if tag == b"hvcC" { + let c = &h[i + 4..]; + if c.len() > 23 && c[0] == 1 { + let n = c[22] as usize; + let mut p = 23; + let (mut vps, mut sps, mut pps) = (vec![], vec![], vec![]); + for _ in 0..n { + let t = c.get(p)? & 0x3F; + let cnt = u16::from_be_bytes([*c.get(p + 1)?, *c.get(p + 2)?]) as usize; + p += 3; + for _ in 0..cnt { + let l = u16::from_be_bytes([*c.get(p)?, *c.get(p + 1)?]) as usize; + let nal = c.get(p + 2..p + 2 + l)?.to_vec(); + match t { + 32 => vps.push(nal), + 33 => sps.push(nal), + 34 => pps.push(nal), + _ => {} + } + p += 2 + l; + } + } + if !sps.is_empty() && !pps.is_empty() { + return Some(MediaConfig { codec: Codec::Hevc, sps, pps, vps }); + } + } + } + } + } + None +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Codec { + Avc, + Hevc, +} + +#[derive(Clone, Debug)] +struct Sample { + /// Posição absoluta (na origem) do primeiro byte da amostra (prefixo de tamanho incluso). + off: u64, + size: u32, + sync: bool, + poc: Option, + idr: bool, +} + +pub struct Rebuilt { + /// Bytes do arquivo original antes do `mdat` (ftyp etc.). + pub pre_len: u64, + /// Onde começam e terminam os dados dos quadros na origem. + pub payload: (u64, u64), + pub moov: Vec, + pub frames: usize, + pub width: u32, + pub height: u32, + pub codec: Codec, + pub fps: f64, + pub skipped_bytes: u64, +} + +impl Rebuilt { + /// Cabeçalho `mdat` de 64 bits para o payload. + pub fn mdat_header(&self) -> Vec { + let size = 16 + (self.payload.1 - self.payload.0); + let mut v = Vec::with_capacity(16); + v.extend_from_slice(&1u32.to_be_bytes()); + v.extend_from_slice(b"mdat"); + v.extend_from_slice(&size.to_be_bytes()); + v + } +} + +fn avc_nal_ok(b: u8) -> bool { + b & 0x80 == 0 && matches!(b & 0x1F, 1 | 5..=12) +} +fn hevc_nal_ok(b0: u8, b1: u8) -> bool { + b0 & 0x81 == 0 && ((b0 >> 1) & 0x3F) <= 40 && b1 & 0x07 != 0 && (b1 >> 3) == 0 +} + +/// Cabeçalho do átomo `mdat` que começa em `at`: (início dos dados, tamanho declarado ou 0). +fn mdat_at(src: &dyn Source, at: u64) -> Option<(u64, u64)> { + let h = read_vec(src, at, 16).ok()?; + if h.len() < 8 || &h[4..8] != b"mdat" { + return None; + } + let sz = be32(&h, 0) as u64; + if sz == 1 && h.len() >= 16 { + Some((at + 16, u64::from_be_bytes(h[8..16].try_into().unwrap()))) + } else { + Some((at + 8, sz)) + } +} + +/// Reconstrói o índice de um MP4 que começa em `off`, sem `moov`. `avail` limita a leitura. +pub fn rebuild(src: &dyn Source, off: u64, avail: u64, refs: &[MediaConfig]) -> Option { + if let Some(r) = rebuild_with(src, off, avail, None) { + return Some(r); + } + refs.iter().find_map(|c| rebuild_with(src, off, avail, Some(c))) +} + +fn rebuild_with(src: &dyn Source, off: u64, avail: u64, preset: Option<&MediaConfig>) -> Option { + // Átomos antes do mdat. + let mut p = 0u64; + let mut pre_end = None; + for _ in 0..16 { + let h = read_vec(src, off + p, 16).ok()?; + if h.len() < 8 { + return None; + } + let ty = &h[4..8]; + if ty == b"mdat" { + pre_end = Some(p); + break; + } + if !matches!(ty, b"ftyp" | b"free" | b"skip" | b"wide" | b"uuid" | b"pnot" | b"junk" | b"styp") { + return None; + } + let sz = be32(&h, 0) as u64; + if sz < 8 { + return None; + } + p += sz; + } + let pre_len = pre_end?; + let (data_start, declared) = mdat_at(src, off + pre_len)?; + let hard_end = off + avail.min(src.size() - off); + let data_end_decl = if declared == 0 { hard_end } else { (off + pre_len + declared).min(hard_end) }; + + // Lê o payload em blocos, interpretando NALs. + let mut samples: Vec = vec![]; + let mut cache: Vec = vec![]; + let mut cache_off = data_start; + let read_at = |o: u64, n: usize, cache: &mut Vec, cache_off: &mut u64| -> Option> { + if o < *cache_off || o + n as u64 > *cache_off + cache.len() as u64 { + let want = (data_end_decl.saturating_sub(o) as usize).min(8 << 20).max(n).min((hard_end - o) as usize); + *cache = read_vec(src, o, want).ok()?; + *cache_off = o; + } + let s = (o - *cache_off) as usize; + cache.get(s..s + n).map(|x| x.to_vec()) + }; + // Detecta o codec: o que forma a cadeia mais longa de unidades NAL válidas. + let chain = |c: Codec| -> usize { + let Ok(head) = read_vec(src, data_start, 64 << 10) else { return 0 }; + let mut p = 0usize; + let mut n = 0; + while p + 6 <= head.len() { + let l = be32(&head, p) as usize; + let ok = l >= 1 && l <= 8 << 20 && match c { + Codec::Avc => avc_nal_ok(head[p + 4]), + Codec::Hevc => hevc_nal_ok(head[p + 4], head[p + 5]), + }; + if !ok { + break; + } + n += 1; + p += 4 + l; + } + n + }; + let codec = match preset { + Some(c) => Some(c.codec), + None => { + let (a, h) = (chain(Codec::Avc), chain(Codec::Hevc)); + if a == 0 && h == 0 { + None + } else if a >= h { + Some(Codec::Avc) + } else { + Some(Codec::Hevc) + } + } + }; + let codec = codec?; + let mut pos = data_start; + let mut payload_end = data_start; + let mut sps_nals: Vec> = vec![]; + let mut pps_nals: Vec> = vec![]; + let mut vps_nals: Vec> = vec![]; + let mut sps264: Option = None; + let mut sps265: Option = None; + let mut pic_order_present = false; + if let Some(c) = preset { + sps_nals = c.sps.clone(); + pps_nals = c.pps.clone(); + vps_nals = c.vps.clone(); + match codec { + Codec::Avc => { + sps264 = c.sps.iter().find_map(|n| parse_sps264(n)); + pic_order_present = c.pps.first().map(|p| pps_pic_order_present(p)).unwrap_or(false); + } + Codec::Hevc => sps265 = c.sps.iter().find_map(|n| parse_sps_hevc(n)), + } + } + let mut idr_total = 0usize; + let mut idr_bad = 0usize; + let mut cur: Option = None; + let mut group_start: Option = None; // início do grupo de NALs não-VCL que precede um quadro + let mut nal_count = 0usize; + let mut max_len = 0u64; + // POC (H.264, tipo 0) + let mut prev_msb: i64 = 0; + let mut prev_lsb: i64 = 0; + + 'outer: while pos + 6 <= data_end_decl { + let hdr = read_at(pos, 6, &mut cache, &mut cache_off)?; + let len = be32(&hdr, 0) as u64; + let ok = len >= 1 && len <= 16 << 20 && pos + 4 + len <= data_end_decl && match codec { + Codec::Avc => avc_nal_ok(hdr[4]), + Codec::Hevc => hevc_nal_ok(hdr[4], hdr[5]), + }; + let mut ok = ok; + if ok && nal_count >= 3 && len > (max_len * 6).max(1 << 20) { + // Um "tamanho" muito maior que os já vistos é lixo (áudio lido como vídeo). + ok = false; + } + if ok { + let head = read_at(pos + 4, (len as usize).min(48), &mut cache, &mut cache_off)?; + ok = nal_plausible(codec, &head, len, sps264.as_ref(), pps_nals.len()); + } + let valid = ok; + if valid { + max_len = max_len.max(len); + } + if !valid { + // Pode ser áudio intercalado: procura a próxima unidade NAL plausível. + let found = scan_nal(src, codec, pos + 1, (pos + (2 << 20)).min(data_end_decl), data_end_decl, sps264.as_ref(), pps_nals.len(), (max_len * 6).max(1 << 20)); + match found { + Some(q) if nal_count > 0 => { + // O que veio antes fecha o quadro atual. + if let Some(mut s) = cur.take() { + s.size = (group_start.unwrap_or(pos) - s.off) as u32; + samples.push(s); + } + group_start = None; + pos = q; + continue 'outer; + } + _ => break 'outer, + } + } + let nal_start = pos + 4; + let nal = read_at(nal_start, len.min(64 << 10) as usize, &mut cache, &mut cache_off)?; + let (is_vcl, first_of_pic, is_idr, ty) = match codec { + Codec::Avc => { + let t = nal[0] & 0x1F; + match t { + 1 | 2 | 5 => { + let first = sps264.as_ref().and_then(|s| slice264(&nal, s, pic_order_present)); + (true, first.as_ref().map(|f| f.first_mb0).unwrap_or(true), t == 5, t) + } + 7 => { + if sps264.is_none() { + sps264 = parse_sps264(&nal); + } + if !sps_nals.iter().any(|x| *x == nal) && sps_nals.len() < 4 { + sps_nals.push(nal.clone()); + } + (false, false, false, t) + } + 8 => { + if pps_nals.is_empty() { + pic_order_present = pps_pic_order_present(&nal); + } + if !pps_nals.iter().any(|x| *x == nal) && pps_nals.len() < 4 { + pps_nals.push(nal.clone()); + } + (false, false, false, t) + } + _ => (false, false, false, t), + } + } + Codec::Hevc => { + let t = (nal[0] >> 1) & 0x3F; + match t { + 0..=9 | 16..=21 => (true, nal.get(2).map(|b| b & 0x80 != 0).unwrap_or(true), (16..=21).contains(&t), t), + 32 => { + if !vps_nals.iter().any(|x| *x == nal) && vps_nals.len() < 2 { + vps_nals.push(nal.clone()); + } + (false, false, false, t) + } + 33 => { + if sps265.is_none() { + sps265 = parse_sps_hevc(&nal); + } + if !sps_nals.iter().any(|x| *x == nal) && sps_nals.len() < 4 { + sps_nals.push(nal.clone()); + } + (false, false, false, t) + } + 34 => { + if !pps_nals.iter().any(|x| *x == nal) && pps_nals.len() < 4 { + pps_nals.push(nal.clone()); + } + (false, false, false, t) + } + _ => (false, false, false, t), + } + } + }; + let _ = ty; + if is_vcl { + if first_of_pic { + // fecha o quadro anterior + if let Some(mut s) = cur.take() { + // tamanho até o início do grupo atual (ou deste NAL) + let end = group_start.unwrap_or(pos); + s.size = (end - s.off) as u32; + samples.push(s); + } + let start = group_start.take().unwrap_or(pos); + let mut poc = None; + if codec == Codec::Avc { + if let Some(sp) = &sps264 { + if let Some(si) = slice264(&nal, sp, pic_order_present) { + if si.idr { + prev_msb = 0; + prev_lsb = 0; + idr_total += 1; + if si.frame_num != 0 { + idr_bad += 1; + } + } + if let Some(lsb) = si.poc_lsb { + let max = 1i64 << sp.log2_max_poc_lsb; + let lsb = lsb as i64; + let msb = if lsb < prev_lsb && prev_lsb - lsb >= max / 2 { + prev_msb + max + } else if lsb > prev_lsb && lsb - prev_lsb > max / 2 { + prev_msb - max + } else { + prev_msb + }; + poc = Some(msb + lsb); + if nal[0] & 0x60 != 0 { + prev_msb = msb; + prev_lsb = lsb; + } + let _ = si.bottom; + } + } + } + } + cur = Some(Sample { off: start, size: 0, sync: is_idr, poc, idr: is_idr }); + } + if !first_of_pic { + group_start = None; + } + } else if cur.is_some() && group_start.is_none() { + group_start = Some(pos); + } else if cur.is_none() && group_start.is_none() { + group_start = Some(pos); + } + pos = nal_start + len; + nal_count += 1; + payload_end = pos; + } + if let Some(mut s) = cur.take() { + s.size = (payload_end - s.off) as u32; + samples.push(s); + } + samples.retain(|s| s.size > 4); + if samples.len() < 2 || (idr_total > 0 && idr_bad * 4 > idr_total) { + return None; + } + let (width, height, timing, profile_info) = match codec { + Codec::Avc => { + let s = sps264.clone()?; + (s.width, s.height, s.timing, None::) + } + Codec::Hevc => { + let s = sps265.clone()?; + (s.width, s.height, s.timing, Some(s)) + } + }; + if width == 0 || height == 0 || sps_nals.is_empty() || pps_nals.is_empty() { + return None; + } + let (num, den) = match timing { + Some((units, scale)) if codec == Codec::Avc => (scale, 2 * units), // quadros por segundo = scale / (2 * units) + Some((units, scale)) => (scale, units), + None => (30, 1), + }; + let fps = num as f64 / den as f64; + let (num, den) = if !(1.0..=240.0).contains(&fps) { (30, 1) } else { (num, den) }; + let fps = num as f64 / den as f64; + // Escala: tique do quadro = den, escala de tempo = num. + let (timescale, dur) = (num, den); + + // Ordem de apresentação (POC) por trecho entre IDR. + let mut ctts: Option> = None; + if samples.iter().all(|s| s.poc.is_some()) { + let mut offs = vec![0i64; samples.len()]; + let mut seg_start = 0; + let mut any = false; + let mut i = 0; + while i <= samples.len() { + if i == samples.len() || (i > seg_start && samples[i].idr) { + let mut order: Vec = (seg_start..i).collect(); + order.sort_by_key(|&k| samples[k].poc.unwrap()); + for (rank, &k) in order.iter().enumerate() { + let d = rank as i64 - (k - seg_start) as i64; + offs[k] = d; + any |= d != 0; + } + seg_start = i; + } + i += 1; + } + if any { + let depth = offs.iter().map(|&d| -d).max().unwrap_or(0).max(0); + ctts = Some(offs.iter().map(|&d| (d + depth) * dur as i64).collect()); + } + } + let reorder_delay: i64 = ctts.as_ref().map(|c| c.iter().copied().min().unwrap_or(0)).unwrap_or(0); + + let payload = (data_start, payload_end); + let out_payload_start = pre_len + 16; + let map_off = |abs: u64| -> u64 { out_payload_start + (abs - data_start) }; + let moov = build_moov(codec, width, height, &sps_nals, &pps_nals, &vps_nals, profile_info.as_ref(), &sps264, &samples, timescale, dur, ctts.as_deref(), reorder_delay, &map_off); + let used: u64 = samples.iter().map(|s| s.size as u64).sum(); + Some(Rebuilt { + pre_len, + payload, + moov, + frames: samples.len(), + width, + height, + codec, + fps, + skipped_bytes: (payload_end - data_start).saturating_sub(used), + }) +} + +// -------------------------------------------------------------------- boxes + +fn bx(ty: &[u8; 4], body: &[u8]) -> Vec { + let mut v = Vec::with_capacity(8 + body.len()); + v.extend_from_slice(&((8 + body.len()) as u32).to_be_bytes()); + v.extend_from_slice(ty); + v.extend_from_slice(body); + v +} + +fn full(ty: &[u8; 4], version: u8, flags: u32, body: &[u8]) -> Vec { + let mut b = vec![version]; + b.extend_from_slice(&flags.to_be_bytes()[1..]); + b.extend_from_slice(body); + bx(ty, &b) +} + +fn u32b(v: u32) -> [u8; 4] { + v.to_be_bytes() +} + +#[allow(clippy::too_many_arguments)] +fn build_moov( + codec: Codec, + width: u32, + height: u32, + sps: &[Vec], + pps: &[Vec], + vps: &[Vec], + hevc: Option<&SpsHevc>, + avc: &Option, + samples: &[Sample], + timescale: u32, + dur: u32, + ctts: Option<&[i64]>, + reorder_delay: i64, + map_off: &dyn Fn(u64) -> u64, +) -> Vec { + let n = samples.len(); + let media_dur = n as u64 * dur as u64; + let movie_ts = 1000u32; + let movie_dur = media_dur * movie_ts as u64 / timescale as u64; + + // Entrada de amostra + let mut entry = vec![]; + entry.extend_from_slice(&[0; 6]); + entry.extend_from_slice(&1u16.to_be_bytes()); + entry.extend_from_slice(&[0; 16]); + entry.extend_from_slice(&(width as u16).to_be_bytes()); + entry.extend_from_slice(&(height as u16).to_be_bytes()); + entry.extend_from_slice(&0x0048_0000u32.to_be_bytes()); + entry.extend_from_slice(&0x0048_0000u32.to_be_bytes()); + entry.extend_from_slice(&0u32.to_be_bytes()); + entry.extend_from_slice(&1u16.to_be_bytes()); + entry.extend_from_slice(&[0; 32]); + entry.extend_from_slice(&0x0018u16.to_be_bytes()); + entry.extend_from_slice(&0xFFFFu16.to_be_bytes()); + let (fourcc, cfg) = match codec { + Codec::Avc => { + let s = avc.as_ref().unwrap(); + let mut c = vec![1, s.profile, s.compat, s.level, 0xFF, 0xE0 | sps.len() as u8]; + for x in sps { + c.extend_from_slice(&(x.len() as u16).to_be_bytes()); + c.extend_from_slice(x); + } + c.push(pps.len() as u8); + for x in pps { + c.extend_from_slice(&(x.len() as u16).to_be_bytes()); + c.extend_from_slice(x); + } + (*b"avc1", bx(b"avcC", &c)) + } + Codec::Hevc => { + let h = hevc.unwrap(); + let mut c = vec![1]; + c.extend_from_slice(&h.ptl[..12.min(h.ptl.len())]); // espaço/tier/perfil, compat, restrições, nível + c.truncate(13); + while c.len() < 13 { + c.push(0); + } + c.extend_from_slice(&0xF000u16.to_be_bytes()); // segmentação espacial + c.push(0xFC); // paralelismo + c.push(0xFC | h.chroma as u8); + c.push(0xF8 | (h.bit_depth_luma - 8) as u8); + c.push(0xF8 | (h.bit_depth_chroma - 8) as u8); + c.extend_from_slice(&0u16.to_be_bytes()); // avgFrameRate + c.push(0x0B); // constFrameRate=0, 1 camada temporal, lengthSizeMinusOne=3 + let arrays: [(u8, &[Vec]); 3] = [(32, vps), (33, sps), (34, pps)]; + let present: Vec<_> = arrays.iter().filter(|(_, v)| !v.is_empty()).collect(); + c.push(present.len() as u8); + for (t, v) in present { + c.push(0x80 | t); + c.extend_from_slice(&(v.len() as u16).to_be_bytes()); + for x in v.iter() { + c.extend_from_slice(&(x.len() as u16).to_be_bytes()); + c.extend_from_slice(x); + } + } + (*b"hvc1", bx(b"hvcC", &c)) + } + }; + entry.extend_from_slice(&cfg); + let sample_entry = bx(&fourcc, &entry); + let mut stsd_body = u32b(1).to_vec(); + stsd_body.extend_from_slice(&sample_entry); + let stsd = full(b"stsd", 0, 0, &stsd_body); + + let stts = full(b"stts", 0, 0, &[u32b(1).as_slice(), &u32b(n as u32), &u32b(dur)].concat()); + let mut stsz_body = u32b(0).to_vec(); + stsz_body.extend_from_slice(&u32b(n as u32)); + for s in samples { + stsz_body.extend_from_slice(&u32b(s.size)); + } + let stsz = full(b"stsz", 0, 0, &stsz_body); + let stsc = full(b"stsc", 0, 0, &[u32b(1).as_slice(), &u32b(1), &u32b(1), &u32b(1)].concat()); + let need64 = samples.iter().any(|s| map_off(s.off) > u32::MAX as u64); + let stco = if need64 { + let mut b = u32b(n as u32).to_vec(); + for s in samples { + b.extend_from_slice(&map_off(s.off).to_be_bytes()); + } + full(b"co64", 0, 0, &b) + } else { + let mut b = u32b(n as u32).to_vec(); + for s in samples { + b.extend_from_slice(&u32b(map_off(s.off) as u32)); + } + full(b"stco", 0, 0, &b) + }; + let syncs: Vec = samples.iter().enumerate().filter(|(_, s)| s.sync).map(|(i, _)| i as u32 + 1).collect(); + let mut stbl_parts = vec![stsd, stts]; + if let Some(c) = ctts { + // entradas agrupadas + let mut runs: Vec<(u32, u32)> = vec![]; + for &v in c { + match runs.last_mut() { + Some(l) if l.1 == v as u32 => l.0 += 1, + _ => runs.push((1, v as u32)), + } + } + let mut b = u32b(runs.len() as u32).to_vec(); + for (cnt, v) in runs { + b.extend_from_slice(&u32b(cnt)); + b.extend_from_slice(&u32b(v)); + } + stbl_parts.push(full(b"ctts", 0, 0, &b)); + } + if !syncs.is_empty() && syncs.len() < n { + let mut b = u32b(syncs.len() as u32).to_vec(); + for s in &syncs { + b.extend_from_slice(&u32b(*s)); + } + stbl_parts.push(full(b"stss", 0, 0, &b)); + } + stbl_parts.push(stsc); + stbl_parts.push(stsz); + stbl_parts.push(stco); + let stbl = bx(b"stbl", &stbl_parts.concat()); + + let url = full(b"url ", 0, 1, &[]); + let dref = full(b"dref", 0, 0, &[u32b(1).as_slice(), &url].concat()); + let dinf = bx(b"dinf", &dref); + let vmhd = full(b"vmhd", 0, 1, &[0; 8]); + let minf = bx(b"minf", &[vmhd, dinf, stbl].concat()); + let mut mdhd_body = vec![]; + mdhd_body.extend_from_slice(&[0; 8]); // criação/modificação + mdhd_body.extend_from_slice(&u32b(timescale)); + mdhd_body.extend_from_slice(&u32b(media_dur as u32)); + mdhd_body.extend_from_slice(&0x55C4u16.to_be_bytes()); // "und" + mdhd_body.extend_from_slice(&[0; 2]); + let mdhd = full(b"mdhd", 0, 0, &mdhd_body); + let mut hdlr_body = vec![0; 4]; + hdlr_body.extend_from_slice(b"vide"); + hdlr_body.extend_from_slice(&[0; 12]); + hdlr_body.extend_from_slice(b"Guara\0"); + let hdlr = full(b"hdlr", 0, 0, &hdlr_body); + let mdia = bx(b"mdia", &[mdhd, hdlr, minf].concat()); + + let matrix: [u32; 9] = [0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x4000_0000]; + let mut tkhd_body = vec![]; + tkhd_body.extend_from_slice(&[0; 8]); + tkhd_body.extend_from_slice(&u32b(1)); // track id + tkhd_body.extend_from_slice(&[0; 4]); + tkhd_body.extend_from_slice(&u32b(movie_dur as u32)); + tkhd_body.extend_from_slice(&[0; 8]); + tkhd_body.extend_from_slice(&[0; 4]); // camada, grupo + tkhd_body.extend_from_slice(&[0; 4]); // volume, reservado + for m in matrix { + tkhd_body.extend_from_slice(&u32b(m)); + } + tkhd_body.extend_from_slice(&u32b(width << 16)); + tkhd_body.extend_from_slice(&u32b(height << 16)); + let tkhd = full(b"tkhd", 0, 3, &tkhd_body); + let mut trak_parts = vec![tkhd]; + if reorder_delay > 0 { + let media_time = reorder_delay; + let mut e = u32b(1).to_vec(); + e.extend_from_slice(&u32b(movie_dur as u32)); + e.extend_from_slice(&u32b(media_time as u32)); + e.extend_from_slice(&0x0001_0000u32.to_be_bytes()); + trak_parts.push(bx(b"edts", &full(b"elst", 0, 0, &e))); + } + trak_parts.push(mdia); + let trak = bx(b"trak", &trak_parts.concat()); + + let mut mvhd_body = vec![]; + mvhd_body.extend_from_slice(&[0; 8]); + mvhd_body.extend_from_slice(&u32b(movie_ts)); + mvhd_body.extend_from_slice(&u32b(movie_dur as u32)); + mvhd_body.extend_from_slice(&0x0001_0000u32.to_be_bytes()); + mvhd_body.extend_from_slice(&0x0100u16.to_be_bytes()); + mvhd_body.extend_from_slice(&[0; 10]); + for m in matrix { + mvhd_body.extend_from_slice(&u32b(m)); + } + mvhd_body.extend_from_slice(&[0; 24]); + mvhd_body.extend_from_slice(&u32b(2)); + let mvhd = full(b"mvhd", 0, 0, &mvhd_body); + bx(b"moov", &[mvhd, trak].concat()) +} + +/// A unidade NAL parece de verdade? Valida o cabeçalho de fatia e o tamanho de SPS/PPS/AUD. +fn nal_plausible(codec: Codec, head: &[u8], len: u64, sps: Option<&Sps264>, pps_count: usize) -> bool { + match codec { + Codec::Avc => { + let t = head[0] & 0x1F; + match t { + 1 | 2 | 5 => { + let rbsp = unescape(&head[1..]); + let mut b = Bits::new(&rbsp); + let (Some(first), Some(st), Some(pps)) = (b.ue(), b.ue(), b.ue()) else { return false }; + if st > 9 || pps > 255 || (pps_count > 0 && pps as usize >= pps_count.max(1) && pps > 3) { + return false; + } + if t == 5 && !matches!(st, 2 | 4 | 7 | 9) { + return false; + } + if first > 8191 { + return false; + } + if let Some(sp) = sps { + // frame_num de IDR é sempre 0 + if t == 5 { + if let Some(fnum) = b.u(sp.log2_max_frame_num) { + return fnum == 0; + } + } + } + true + } + 9 => len == 2, + 7 => (4..=300).contains(&len), + 8 => (2..=300).contains(&len), + 6 => len >= 2, + _ => true, + } + } + Codec::Hevc => { + let t = (head[0] >> 1) & 0x3F; + match t { + 35 => len <= 4, + 32 | 33 | 34 => (4..=1000).contains(&len), + _ => true, + } + } + } +} + +/// Quantas unidades NAL válidas seguidas (sem vãos) começam em `at` (até `max`). +fn chain_len(buf: &[u8], base: u64, at: u64, codec: Codec, sps: Option<&Sps264>, pps_count: usize, cap: u64, max: usize) -> usize { + let mut p = (at - base) as usize; + let mut n = 0; + while n < max && p + 8 <= buf.len() { + let l = be32(buf, p) as u64; + let ok = l >= 1 && l <= cap && p as u64 + 4 + l <= buf.len() as u64 + 8 + && match codec { + Codec::Avc => avc_nal_ok(buf[p + 4]), + Codec::Hevc => hevc_nal_ok(buf[p + 4], buf[p + 5]), + } + && nal_plausible(codec, &buf[p + 4..(p + 4 + (l as usize).min(48)).min(buf.len())], l, sps, pps_count); + if !ok { + break; + } + n += 1; + p += 4 + l as usize; + } + n +} + +/// Próxima posição em [from, limit) onde começa uma sequência de unidades NAL plausíveis. +/// Prefere o primeiro ponto com duas ou mais unidades seguidas; senão, uma unidade isolada. +fn scan_nal(src: &dyn Source, codec: Codec, from: u64, limit: u64, data_end: u64, sps: Option<&Sps264>, pps_count: usize, cap: u64) -> Option { + if limit <= from + 8 { + return None; + } + // Lê um pouco além do limite para poder seguir a cadeia de unidades. + let want = ((limit - from) as usize + (cap as usize).min(4 << 20)).min((data_end - from) as usize); + let buf = read_vec(src, from, want).ok()?; + let mut single = None; + let mut i = 0usize; + while i + 8 < buf.len() && from + (i as u64) < limit { + let q = from + i as u64; + let l2 = be32(&buf, i) as u64; + if l2 >= 2 && l2 <= cap && q + 4 + l2 <= data_end { + let nb = buf[i + 4]; + let good_head = match codec { + // Um quadro novo começa por parâmetros/AUD/SEI ou por fatia do macrobloco 0. + Codec::Avc => avc_nal_ok(nb) && (matches!(nb & 0x1F, 6..=9) || (matches!(nb & 0x1F, 1 | 5) && buf[i + 5] & 0x80 != 0)), + Codec::Hevc => hevc_nal_ok(nb, buf[i + 5]), + }; + if good_head { + let c = chain_len(&buf, from, q, codec, sps, pps_count, cap, 3); + if c >= 2 { + return Some(q); + } + if c == 1 && single.is_none() { + single = Some(q); + } + } + } + i += 1; + } + single +} diff --git a/crates/guara-core/src/recover.rs b/crates/guara-core/src/recover.rs index ac2cc4c..32373a9 100644 --- a/crates/guara-core/src/recover.rs +++ b/crates/guara-core/src/recover.rs @@ -375,6 +375,29 @@ pub fn write_file(src: &dyn Source, f: &Found, path: &Path) -> Result { out.write_all(bytes)?; hasher.update(bytes); } + DataLoc::Parts { parts } => { + for part in parts { + match part { + Part::Bytes { bytes } => { + out.write_all(bytes)?; + hasher.update(bytes); + } + Part::Run { offset, len } => { + let mut done = 0u64; + while done < *len { + let n = (*len - done).min(1 << 20) as usize; + let got = read_vec(src, offset + done, n)?; + if got.len() < n { + return Err(Error::msg("leitura além do fim da origem")); + } + out.write_all(&got)?; + hasher.update(&got); + done += n as u64; + } + } + } + } + } DataLoc::Extents { extents } => { let mut left = f.size; let mut buf = vec![0u8; 1 << 20]; @@ -425,6 +448,25 @@ pub fn read_found_at( buf[..n].copy_from_slice(&bytes[o..o + n]); Ok(n) } + DataLoc::Parts { parts } => { + let mut pos = 0u64; + for p in parts { + let plen = p.len(); + if offset < pos + plen { + let within = offset - pos; + let n = want.min((plen - within) as usize); + return match p { + Part::Bytes { bytes } => { + buf[..n].copy_from_slice(&bytes[within as usize..within as usize + n]); + Ok(n) + } + Part::Run { offset: o, .. } => src.read_at(o + within, &mut buf[..n]), + }; + } + pos += plen; + } + Ok(0) + } DataLoc::Extents { extents } => { let mut pos = 0u64; for e in extents { diff --git a/crates/guara-core/src/scan.rs b/crates/guara-core/src/scan.rs index e14050d..342f4d1 100644 --- a/crates/guara-core/src/scan.rs +++ b/crates/guara-core/src/scan.rs @@ -207,6 +207,15 @@ pub fn run( total, index: crate::defrag::ClusterIndex::new(src.size()), pending: vec![], + intact_mp4: vec![], + refs: opts + .references + .iter() + .filter_map(|p| { + let s = crate::source::open_path(std::path::Path::new(p)).ok()?; + crate::mp4rebuild::config_from_mp4(s.as_ref(), 0, s.size()) + }) + .collect(), carved: vec![], }; for ri in 0..cp.ranges.len() { @@ -227,10 +236,11 @@ pub fn run( Outcome::Completed } -/// JPEG que parece quebrado: tenta recompor depois de varrer o disco todo. -struct PendingJpeg { +/// Arquivo que parece quebrado: tenta recompor depois de varrer o disco todo. +struct Pending { found: Found, hit_len: u64, + mp4: bool, } struct Handler<'a> { @@ -242,16 +252,58 @@ struct Handler<'a> { done: u64, total: u64, index: crate::defrag::ClusterIndex, - pending: Vec, + pending: Vec, + /// Vídeos íntegros achados no carving (início, tamanho): fontes de SPS/PPS. + intact_mp4: Vec<(u64, u64)>, + refs: Vec, /// Extents de arquivos carved já emitidos (não podem servir de fragmento para outro). carved: Vec, } impl Handler<'_> { + fn rebuild_mp4(&mut self, p: Pending, off: u64) { + // Referências: as informadas pelo usuário e os vídeos íntegros mais próximos no disco. + let mut refs = self.refs.clone(); + let mut near = self.intact_mp4.clone(); + near.sort_by_key(|&(o, _)| o.abs_diff(off)); + for (o, l) in near.into_iter().take(6) { + if let Some(c) = crate::mp4rebuild::config_from_mp4(self.src, o, l) { + refs.push(c); + } + } + let Some(r) = crate::mp4rebuild::rebuild(self.src, off, p.hit_len, &refs) else { return }; + let Ok(pre) = crate::source::read_vec(self.src, off, r.pre_len as usize) else { return }; + let payload_len = r.payload.1 - r.payload.0; + let mut f = p.found; + let hdr = r.mdat_header(); + f.size = pre.len() as u64 + hdr.len() as u64 + payload_len + r.moov.len() as u64; + f.data = DataLoc::Parts { + parts: vec![ + Part::Bytes { bytes: pre }, + Part::Bytes { bytes: hdr }, + Part::Run { offset: r.payload.0, len: payload_len }, + Part::Bytes { bytes: r.moov }, + ], + }; + f.ext = "mp4".into(); + f.name = format!("arquivo_{:06}.mp4", f.id); + f.confidence = Confidence::Medium; + f.note = Some(format!( + "índice (moov) reconstruído: {} quadros {}x{} a {:.1} fps, sem áudio; ajuste de tempo aproximado", + r.frames, r.width, r.height, r.fps + )); + self.carved.push(Extent::new(r.payload.0, payload_len)); + self.sink.replace(f); + } + fn rebuild_pending(&mut self) { let pend = std::mem::take(&mut self.pending); for p in pend { let off = p.found.first_offset().unwrap_or(0); + if p.mp4 { + self.rebuild_mp4(p, off); + continue; + } let mut used = self.fs_extents.clone(); used.extend(self.carved.iter().copied()); // O próprio começo do arquivo não conta como "ocupado" para ele mesmo. @@ -314,6 +366,15 @@ impl CarveEvents for Handler<'_> { crate::defrag::Analysis::Unsupported | crate::defrag::Analysis::Rebuilt { .. } => {} } } + let mut pending_mp4 = false; + if matches!(hit.ext, "mp4" | "mov" | "m4v" | "3gp") { + if hit.confidence == Confidence::High { + self.intact_mp4.push((off, hit.len)); + } else { + pending_mp4 = true; + pending_len = Some(hit.len); + } + } let hit = &hit; let id = self .next_id @@ -348,8 +409,8 @@ impl CarveEvents for Handler<'_> { self.carved.push(Extent::new(off, hit.len)); if let Some(hl) = pending_len { let mut pf = f.clone(); - pf.note = Some("JPEG interrompido ou fragmentado".into()); - self.pending.push(PendingJpeg { found: pf, hit_len: hl }); + pf.note = Some(if pending_mp4 { "vídeo sem índice (moov)".into() } else { "JPEG interrompido ou fragmentado".into() }); + self.pending.push(Pending { found: pf, hit_len: hl, mp4: pending_mp4 }); } self.sink.found(f); match skip_len { diff --git a/crates/guara-core/src/types.rs b/crates/guara-core/src/types.rs index db43449..14437ea 100644 --- a/crates/guara-core/src/types.rs +++ b/crates/guara-core/src/types.rs @@ -106,6 +106,72 @@ pub enum DataLoc { Extents { extents: Vec, }, + /// Arquivo montado: trechos em memória (cabeçalhos refeitos) e trechos do disco. + Parts { + parts: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "part", rename_all = "lowercase")] +pub enum Part { + Bytes { + #[serde(with = "b64")] + bytes: Vec, + }, + /// Trecho contíguo da origem. + Run { offset: u64, len: u64 }, +} + +impl Part { + pub fn len(&self) -> u64 { + match self { + Part::Bytes { bytes } => bytes.len() as u64, + Part::Run { len, .. } => *len, + } + } + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +mod b64 { + use serde::{Deserialize, Deserializer, Serializer}; + const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + pub fn serialize(b: &Vec, s: S) -> Result { + let mut out = String::with_capacity(b.len().div_ceil(3) * 4); + for c in b.chunks(3) { + let n = (c[0] as u32) << 16 | (*c.get(1).unwrap_or(&0) as u32) << 8 | *c.get(2).unwrap_or(&0) as u32; + out.push(T[(n >> 18) as usize & 63] as char); + out.push(T[(n >> 12) as usize & 63] as char); + out.push(if c.len() > 1 { T[(n >> 6) as usize & 63] as char } else { '=' }); + out.push(if c.len() > 2 { T[n as usize & 63] as char } else { '=' }); + } + s.serialize_str(&out) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + let mut out = vec![]; + let (mut acc, mut bits) = (0u32, 0); + for c in s.bytes() { + let v = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' => 62, + b'/' => 63, + _ => continue, + } as u32; + acc = (acc << 6) | v; + bits += 6; + if bits >= 8 { + bits -= 8; + out.push((acc >> bits) as u8); + acc &= (1 << bits) - 1; + } + } + Ok(out) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -188,6 +254,10 @@ impl Found { match &self.data { DataLoc::Extents { extents } => extents.iter().find(|e| !e.sparse).map(|e| e.offset), DataLoc::Resident { .. } => None, + DataLoc::Parts { parts } => parts.iter().find_map(|p| match p { + Part::Run { offset, .. } => Some(*offset), + _ => None, + }), } } @@ -330,6 +400,9 @@ pub struct ScanOptions { /// O disco principal é só membro de RAID sem metadados: varre apenas os volumes virtuais. #[serde(default)] pub skip_primary: bool, + /// Vídeos íntegros da mesma câmera/drone: fornecem SPS/PPS para remontar MP4 sem moov. + #[serde(default)] + pub references: Vec, } fn default_max_carve() -> u64 { @@ -347,6 +420,7 @@ impl Default for ScanOptions { extra_parts: vec![], covered: vec![], skip_primary: false, + references: vec![], } } } diff --git a/crates/guara-core/tests/mp4rebuild.rs b/crates/guara-core/tests/mp4rebuild.rs new file mode 100644 index 0000000..762bb40 --- /dev/null +++ b/crates/guara-core/tests/mp4rebuild.rs @@ -0,0 +1,309 @@ +//! Reconstrução do moov de MP4 H.264: arquivos sintéticos e, se existir, um vídeo real. + +use guara_core::mp4rebuild; +use guara_core::source::MemSource; + +struct BitW { + v: Vec, + n: usize, +} +impl BitW { + fn new() -> Self { + Self { v: vec![], n: 0 } + } + fn bit(&mut self, b: u32) { + if self.n % 8 == 0 { + self.v.push(0); + } + if b != 0 { + *self.v.last_mut().unwrap() |= 0x80 >> (self.n % 8); + } + self.n += 1; + } + fn u(&mut self, val: u32, bits: u32) { + for i in (0..bits).rev() { + self.bit((val >> i) & 1); + } + } + fn ue(&mut self, v: u32) { + let x = v + 1; + let len = 32 - x.leading_zeros(); + for _ in 0..len - 1 { + self.bit(0); + } + self.u(x, len); + } +} + +const SPS: &str = "67f4000d90d9b28283f602d41818190000030001000003003c8f142996"; +const PPS: &str = "68ebec448440"; + +fn hex(s: &str) -> Vec { + (0..s.len() / 2).map(|i| u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).unwrap()).collect() +} + +/// Cabeçalho de fatia H.264 conforme o SPS acima (log2_max_frame_num e log2_max_poc_lsb lidos do SPS). +fn slice(idr: bool, frame_num: u32, poc_lsb: u32, payload: usize, seed: u8, sps: &mguara::Sps) -> Vec { + let mut b = BitW::new(); + b.ue(0); // first_mb_in_slice + b.ue(if idr { 7 } else { 5 }); // slice_type + b.ue(0); // pps + b.u(frame_num, sps.log2_max_frame_num); + if idr { + b.ue(0); + } + b.u(poc_lsb, sps.log2_max_poc_lsb); + b.u(0x55, 8); + let mut nal = vec![if idr { 0x65 } else { 0x41 }]; + nal.extend_from_slice(&b.v); + nal.extend((0..payload).map(|i| (i as u8).wrapping_mul(7).wrapping_add(seed) | 0x10)); + nal +} + +mod mguara { + pub use guara_core::mp4rebuild::Sps264 as Sps; +} + +fn nal(out: &mut Vec, n: &[u8]) { + out.extend_from_slice(&(n.len() as u32).to_be_bytes()); + out.extend_from_slice(n); +} + +/// Monta um MP4 sem moov: ftyp, free, mdat. Devolve (arquivo, tamanhos esperados, sincronos, ordem de exibição). +fn synthetic(gops: usize) -> (Vec, Vec, Vec) { + synthetic_opt(gops, true) +} + +fn synthetic_opt(gops: usize, inband: bool) -> (Vec, Vec, Vec) { + let sps_nal = hex(SPS); + let sps = guara_core::mp4rebuild::parse_sps264(&sps_nal).expect("SPS"); + assert_eq!((sps.width, sps.height), (320, 240)); + let mut mdat = vec![]; + let mut sizes = vec![]; + let mut sync = vec![]; + // GOP: I P B B P B B (ordem de decodificação) com POC de exibição 0 6 2 4 12 8 10 (em unidades de 2) + let pocs = [0u32, 6, 2, 4, 12, 8, 10]; + let mut frame_num = 0; + for g in 0..gops { + for (i, &p) in pocs.iter().enumerate() { + let start = mdat.len(); + nal(&mut mdat, &[0x09, 0x10]); + if i == 0 && inband { + nal(&mut mdat, &sps_nal); + nal(&mut mdat, &hex(PPS)); + } + if i == 0 { + frame_num = 0; + } + let s = slice(i == 0, frame_num % 16, p % 64, 600 + i * 90 + g * 7, (g * 7 + i) as u8, &sps); + nal(&mut mdat, &s); + if p % 4 == 0 || i == 1 || i == 4 { + frame_num += 1; + } + sizes.push(mdat.len() - start); + sync.push(i == 0); + } + } + let mut f = vec![]; + f.extend_from_slice(&[0, 0, 0, 24, b'f', b't', b'y', b'p', b'i', b's', b'o', b'm', 0, 0, 2, 0, b'i', b's', b'o', b'm', b'a', b'v', b'c', b'1']); + f.extend_from_slice(&[0, 0, 0, 8, b'f', b'r', b'e', b'e']); + f.extend_from_slice(&((8 + mdat.len()) as u32).to_be_bytes()); + f.extend_from_slice(b"mdat"); + f.extend_from_slice(&mdat); + (f, sizes, sync) +} + +fn assemble(file: &[u8], r: &mp4rebuild::Rebuilt) -> Vec { + let mut out = file[..r.pre_len as usize].to_vec(); + out.extend_from_slice(&r.mdat_header()); + out.extend_from_slice(&file[r.payload.0 as usize..r.payload.1 as usize]); + out.extend_from_slice(&r.moov); + out +} + +#[test] +fn rebuilds_synthetic_stream_with_b_frames_and_garbage_tail() { + let (mut file, sizes, sync) = synthetic(6); + let real_len = file.len(); + // Lixo depois do fim real (outro arquivo no disco) e tamanho do mdat declarado como "até o fim". + file.extend((0..20_000u32).map(|i| (i.wrapping_mul(2654435761) >> 11) as u8 | 0x80)); + let n = file.len() as u32 - 32; + file[32..36].copy_from_slice(&n.to_be_bytes()); + let src = MemSource(file.clone()); + let r = mp4rebuild::rebuild(&src, 0, file.len() as u64, &[]).expect("reconstrução"); + assert_eq!(r.frames, sizes.len()); + assert_eq!((r.width, r.height), (320, 240)); + assert_eq!(r.payload.1 as usize, real_len, "o fim do vídeo foi detectado"); + let out = assemble(&file, &r); + let size = out.len() as u64; + let mut mp4 = mp4::Mp4Reader::read_header(std::io::Cursor::new(out), size).expect("moov válido"); + let track_id = *mp4.tracks().keys().next().unwrap(); + let t = &mp4.tracks()[&track_id]; + assert_eq!(t.sample_count() as usize, sizes.len()); + for i in 0..sizes.len() { + let s = mp4.read_sample(track_id, i as u32 + 1).unwrap().unwrap(); + assert_eq!(s.bytes.len(), sizes[i], "tamanho da amostra {}", i + 1); + assert_eq!(s.is_sync, sync[i], "sincronia da amostra {}", i + 1); + } + // Ordem de exibição: pts = dts + deslocamento deve crescer na ordem do POC. + let mut pts: Vec<(i64, usize)> = vec![]; + for i in 0..sizes.len() { + let s = mp4.read_sample(track_id, i as u32 + 1).unwrap().unwrap(); + pts.push((s.start_time as i64 + s.rendering_offset as i64, i)); + } + let pocs = [0u32, 6, 2, 4, 12, 8, 10]; + for g in 0..6 { + let mut order: Vec = (0..7).collect(); + order.sort_by_key(|&k| pts[g * 7 + k].0); + let mut expect: Vec = (0..7).collect(); + expect.sort_by_key(|&k| pocs[k]); + assert_eq!(order, expect, "ordem de exibição do GOP {g}"); + } +} + +#[test] +fn rebuilds_real_h264_file_like_the_original_index() { + let Ok(path) = std::env::var("GUARA_MP4_SAMPLE") else { + eprintln!("SKIP: defina GUARA_MP4_SAMPLE com um MP4 H.264 para comparar com o índice original"); + return; + }; + let orig = std::fs::read(&path).unwrap(); + // Corta o moov original. + let mut p = 0usize; + let mut cut = orig.len(); + while p + 8 <= orig.len() { + let sz = u32::from_be_bytes(orig[p..p + 4].try_into().unwrap()) as usize; + if &orig[p + 4..p + 8] == b"moov" { + cut = p; + break; + } + p += sz; + } + let file = orig[..cut].to_vec(); + let src = MemSource(file.clone()); + let r = mp4rebuild::rebuild(&src, 0, file.len() as u64, &[]).expect("reconstrução"); + let out = assemble(&file, &r); + let osize = orig.len() as u64; + let mut a = mp4::Mp4Reader::read_header(std::io::Cursor::new(orig), osize).unwrap(); + let rsize = out.len() as u64; + let mut b = mp4::Mp4Reader::read_header(std::io::Cursor::new(out), rsize).unwrap(); + let ta = *a.tracks().keys().next().unwrap(); + let tb = *b.tracks().keys().next().unwrap(); + assert_eq!(a.tracks()[&ta].sample_count(), b.tracks()[&tb].sample_count()); + for i in 1..=a.tracks()[&ta].sample_count() { + let x = a.read_sample(ta, i).unwrap().unwrap(); + let y = b.read_sample(tb, i).unwrap().unwrap(); + assert_eq!(x.bytes, y.bytes, "amostra {i}"); + assert_eq!(x.is_sync, y.is_sync, "sincronia {i}"); + } + eprintln!("{} quadros, {:.2} fps", r.frames, r.fps); +} + +#[test] +fn rebuilds_real_video_with_interleaved_audio() { + // GUARA_MP4_BIG: vídeo real com áudio; o teste corta o moov e os primeiros 40 MB do mdat bastam. + let Ok(path) = std::env::var("GUARA_MP4_BIG") else { + return; + }; + use std::io::Read; + let mut f = std::fs::File::open(&path).unwrap(); + let total = f.metadata().unwrap().len(); + let mut head = vec![0u8; 1 << 20]; + f.read_exact(&mut head).unwrap(); + // átomos no começo + let mut p = 0usize; + let (mut ftyp_end, mut mdat_at) = (0usize, None); + while p + 8 <= head.len() { + let sz = u32::from_be_bytes(head[p..p + 4].try_into().unwrap()) as usize; + match &head[p + 4..p + 8] { + b"ftyp" => ftyp_end = p + sz, + b"mdat" => { + mdat_at = Some(p); + break; + } + _ => {} + } + if sz == 0 { + break; + } + p += sz; + } + let Some(mp) = mdat_at else { + eprintln!("SKIP: mdat fora do primeiro MiB (moov no começo é grande)"); + return; + }; + // Reexecuta com a leitura completa do começo para montar: ftyp + mdat (40 MB). + use std::io::Seek; + let mut f = std::fs::File::open(&path).unwrap(); + f.seek(std::io::SeekFrom::Start(mp as u64)).unwrap(); + let mut mdat = vec![0u8; 40 << 20]; + f.read_exact(&mut mdat).unwrap(); + let mut file = head[..ftyp_end].to_vec(); + file.extend_from_slice(&mdat); + // O tamanho do mdat declarado vai muito além dos 40 MB que temos (arquivo cortado). + let src = MemSource(file.clone()); + // Sem SPS/PPS no mdat, é preciso um vídeo de referência da mesma câmera (aqui, o próprio original). + let cfg = mp4rebuild::config_from_mp4(&MemSource(head.clone()), 0, head.len() as u64).expect("avcC do original"); + assert!(mp4rebuild::rebuild(&src, 0, file.len() as u64, &[]).is_none(), "sem referência não há como montar o avcC"); + let r = mp4rebuild::rebuild(&src, 0, file.len() as u64, &[cfg]).expect("reconstrução"); + let out = assemble(&file, &r); + let osize = out.len() as u64; + let mut b = mp4::Mp4Reader::read_header(std::io::Cursor::new(out), osize).unwrap(); + let tb = *b.tracks().keys().next().unwrap(); + // Original (índice completo) para comparar as primeiras amostras. + let f = std::fs::File::open(&path).unwrap(); + let mut a = mp4::Mp4Reader::read_header(std::io::BufReader::new(f), total).unwrap(); + let vid = a.tracks().values().find(|t| t.track_type().map(|x| x == mp4::TrackType::Video).unwrap_or(false)).unwrap().track_id(); + eprintln!("reconstruído: {} quadros, {} bytes pulados (áudio), {}x{}", r.frames, r.skipped_bytes, r.width, r.height); + let sizes_a: Vec = (1..=60).map(|i| a.read_sample(vid, i).unwrap().unwrap().bytes.len()).collect(); + let sizes_b: Vec = (1..=60).map(|i| b.read_sample(tb, i).unwrap().unwrap().bytes.len()).collect(); + eprintln!("ORIG {:?}\nMINE {:?}", &sizes_a[40..60], &sizes_b[40..60]); + let check = (r.frames as u32 - 1).min(400); + for i in 1..=check { + let x = a.read_sample(vid, i).unwrap().unwrap(); + let y = b.read_sample(tb, i).unwrap().unwrap(); + assert_eq!(x.bytes.len(), y.bytes.len(), "tamanho da amostra {i}"); + assert_eq!(x.is_sync, y.is_sync, "sincronia {i}"); + } +} + +mod common; + +#[test] +fn deep_scan_rebuilds_a_video_without_moov_using_a_neighbour_as_reference() { + use guara_core::{ScanMode, ScanOptions}; + common::init(); + // Arquivo íntegro (com moov) da "mesma câmera" e um segundo que perdeu o moov e não traz SPS/PPS. + let (good_raw, _, _) = synthetic_opt(8, true); + let r = mp4rebuild::rebuild(&MemSource(good_raw.clone()), 0, good_raw.len() as u64, &[]).unwrap(); + let good = assemble(&good_raw, &r); + let (broken, sizes, _) = synthetic_opt(10, false); + let mut disk = vec![0u8; 8192]; + let good_off = disk.len(); + disk.extend_from_slice(&good); + disk.resize(disk.len().div_ceil(4096) * 4096 + 8192, 0); + let broken_off = disk.len(); + disk.extend_from_slice(&broken); + disk.extend((0..30_000u32).map(|i| (i.wrapping_mul(2654435761) >> 9) as u8 | 0x80)); + let dir = tempfile::tempdir().unwrap(); + let img = dir.path().join("v.img"); + std::fs::write(&img, &disk).unwrap(); + let c = common::scan_path(&img, &ScanOptions { mode: ScanMode::Deep, ..Default::default() }); + let vids: Vec<_> = c.found.iter().filter(|f| f.ext == "mp4").collect(); + assert_eq!(vids.len(), 2, "{:?}", vids.iter().map(|f| (f.first_offset(), f.size, f.confidence, &f.note)).collect::>()); + let fb = vids.iter().find(|f| matches!(f.data, guara_core::DataLoc::Parts { .. })).expect("vídeo quebrado"); + let _ = broken_off; + assert!(fb.note.as_deref().unwrap_or("").contains("reconstruído"), "{:?}", fb.note); + let _ = good_off; + let out = dir.path().join("out"); + let rep = common::recover_all(&img, std::slice::from_ref(*fb), &out); + assert!(rep.failed.is_empty()); + let p = std::path::PathBuf::from(&rep.recovered[0].path); + let bytes = std::fs::read(if p.is_absolute() { p } else { out.join(p) }).unwrap(); + let len = bytes.len() as u64; + let mut m = mp4::Mp4Reader::read_header(std::io::Cursor::new(bytes), len).expect("MP4 válido"); + let tid = *m.tracks().keys().next().unwrap(); + assert_eq!(m.tracks()[&tid].sample_count() as usize, sizes.len()); + let first = m.read_sample(tid, 1).unwrap().unwrap(); + assert_eq!(first.bytes.len(), sizes[0]); +}