diff --git a/crates/cyber-core/src/binary/editor.rs b/crates/cyber-core/src/binary/editor.rs index c9030d37..b641a2a2 100644 --- a/crates/cyber-core/src/binary/editor.rs +++ b/crates/cyber-core/src/binary/editor.rs @@ -21,10 +21,13 @@ mod catalog; mod changes; #[path = "editor/restore.rs"] mod restore; +#[path = "editor/retention.rs"] +mod retention; #[path = "editor/versions.rs"] mod versions; pub use catalog::{StoredDocument, StoredDocuments}; pub use changes::AcknowledgedChange; +pub use retention::Cleanup; pub use versions::VersionView; const MAX_OPERATIONS: usize = 4096; @@ -126,6 +129,7 @@ pub struct PreparedSave { intent: Intent, after: Draft, expected_revision: String, + _history: File, } pub enum SavePreparation { Existing(Operation), @@ -209,6 +213,7 @@ impl Editor { Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(_) => return Err(Error::Io), } + store.ensure_history_lock()?; catalog::publish_header(&mut store, &owner, document)?; let (base, target) = document.prepare(&store, cancelled)?; if target.root != owner.root { @@ -234,7 +239,7 @@ impl Editor { record, }) } - pub fn resume(store: Store, owner: &Owner) -> Result { + pub fn resume(mut store: Store, owner: &Owner) -> Result { let record: Record = serde_json::from_slice(&store.read_record("document.json")?) .map_err(|_| Error::Invalid)?; if record.version != 1 @@ -244,6 +249,7 @@ impl Editor { { return Err(Error::File); } + store.ensure_history_lock()?; let pins = record.target.pins()?; let draft = Draft::resume(record.base.open(&store)?, record.draft.clone())?; Ok(Self { @@ -507,6 +513,7 @@ impl Editor { after.clone(), &after_draft, self.draft.ranges(), + &stage_name, )?; let intent = Intent { operation, @@ -521,6 +528,7 @@ impl Editor { intent, after: after_draft, expected_revision: self.draft.revision(), + _history: self.store.history_lease(false)?, }))) } fn conflict( @@ -568,7 +576,7 @@ impl Editor { /// explicit newly authorized attempt. It does not manufacture a save receipt. pub fn commit_save_checked( &mut self, - prepared: PreparedSave, + mut prepared: PreparedSave, cancelled: impl Fn() -> bool, authorize: impl FnOnce() -> Result<(), Error>, ) -> Result { @@ -621,6 +629,12 @@ impl Editor { } // Only the pending JSON value changes. Count the prospective publication // by streaming serialization, without cloning the bounded draft history. + // Retention starts at commit preparation, not when a caller first held + // the prepared token. Fix the timestamp before both capacity forecasts + // and intent publication so the reserved outcome shape stays exact. + if let Some(evidence) = &mut prepared.intent.evidence { + evidence.info.created_ms = chrono::Utc::now().timestamp_millis(); + } let pending_size = encoded_size(&Some(&prepared.intent))?; let prior_pending_size = encoded_size(&self.record.pending)?; let intent_bytes = encoded_size(&self.record)? diff --git a/crates/cyber-core/src/binary/editor/restore.rs b/crates/cyber-core/src/binary/editor/restore.rs index 8a7f694c..cc353ac9 100644 --- a/crates/cyber-core/src/binary/editor/restore.rs +++ b/crates/cyber-core/src/binary/editor/restore.rs @@ -148,6 +148,7 @@ impl Editor { after.clone(), &after_draft, ranges, + &stage_name, )?; let intent = Intent { operation, @@ -162,6 +163,7 @@ impl Editor { intent, after: after_draft, expected_revision: self.draft.revision(), + _history: self.store.history_lease(false)?, }))) } } diff --git a/crates/cyber-core/src/binary/editor/retention.rs b/crates/cyber-core/src/binary/editor/retention.rs new file mode 100644 index 00000000..6d1abafa --- /dev/null +++ b/crates/cyber-core/src/binary/editor/retention.rs @@ -0,0 +1,703 @@ +//! Retire only acknowledged history, then reclaim its unreferenced immutable +//! images. Compact tombstones retain candidate references across interruption. +//! Unassociated preparation files and uncertain native stages are never guessed. +use super::*; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Displaced { + name: String, + identity: String, + metadata: native::Metadata, + base: BaseRef, +} +impl Displaced { + pub fn prepared(editor: &Editor, name: &str) -> Self { + Self { + name: name.into(), + identity: editor.record.target.file.clone(), + metadata: editor.record.target.metadata.clone(), + base: editor.record.base.clone(), + } + } +} +#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)] +#[schemars(rename = "BinaryCleanup")] +pub struct Cleanup { + pub versions_retired: usize, + pub bases_removed: usize, + pub stages_removed: usize, + pub stages_retained: usize, + pub preparations_removed: usize, + pub bytes_reclaimed: Offset, +} +impl Default for Cleanup { + fn default() -> Self { + Self { + versions_retired: 0, + bases_removed: 0, + stages_removed: 0, + stages_retained: 0, + preparations_removed: 0, + bytes_reclaimed: Offset(0), + } + } +} +struct Plan { + retire: Vec, + retained: BTreeSet, + candidates: BTreeMap, + stages: Vec, + recognized: BTreeSet, + preparation_contract: bool, +} +struct Stage { + version: String, + ticket: Displaced, + images: [String; 2], +} +fn add_candidate(plan: &mut Plan, base: &BaseRef) -> Result<(), Error> { + if !versions::valid_base(base) { + return Err(Error::Changed); + } + if let Some(prior) = plan.candidates.get(&base.name) + && (prior.length != base.length || prior.sha256 != base.sha256) + { + return Err(Error::Changed); + } + plan.candidates.insert(base.name.clone(), base.clone()); + Ok(()) +} +fn gate( + cancelled: &impl Fn() -> bool, + authorize: &impl Fn() -> Result<(), Error>, +) -> Result<(), Error> { + if cancelled() { + return Err(Error::Cancelled); + } + authorize()?; + if cancelled() { + return Err(Error::Cancelled); + } + Ok(()) +} +impl Editor { + /// Caller supplies the configured retention cutoff and live checkpoint refs, + /// retains native mutation ownership, and refreshes trusted authority before + /// every namespace change. No source bytes are modified by this operation. + pub fn prune_saved_versions( + &mut self, + older_than_ms: i64, + protected_versions: &[String], + cancelled: impl Fn() -> bool, + authorize: impl Fn() -> Result<(), Error>, + ) -> Result { + self.available()?; + if protected_versions.len() > 4096 + || protected_versions + .iter() + .any(|id| !versions::valid_version(id)) + { + return Err(Error::Limit); + } + let _readers = self.store.history_lease(true)?; + let mut report = Cleanup::default(); + if let Some(displaced) = self + .record + .last_save + .as_ref() + .and_then(|last| last.evidence.as_ref()) + .and_then(|evidence| evidence.displaced.clone()) + { + self.reclaim_stage(&displaced, &cancelled, &authorize, &mut report)?; + } + self.publish_last_version_locked()?; + let mut plan = self.retention_plan(older_than_ms, protected_versions, &cancelled)?; + // Resolve known duplicates before retiring their provenance. A changed + // displaced inode keeps the complete version and both exact images. + for stage in &plan.stages { + let retained = report.stages_retained; + self.reclaim_stage(&stage.ticket, &cancelled, &authorize, &mut report)?; + if report.stages_retained > retained { + plan.retained.extend(stage.images.clone()); + plan.retire.retain(|id| id != &stage.version); + } + } + for version in &plan.retire { + if cancelled() { + return Err(Error::Cancelled); + } + let mut evidence = + versions::read_evidence(self.store.directory(), self.store.document_id(), version)?; + gate(&cancelled, &authorize)?; + evidence.retired = true; + evidence.ranges.clear(); + evidence.revision.clear(); + self.store.record( + &format!("{version}.json"), + encoded_size(&evidence)?, + |writer| serde_json::to_writer(writer, &evidence).map_err(|_| Error::Io), + )?; + report.versions_retired += 1; + } + // Only references durably recorded in retired history become candidates. + // An interrupted cleanup can repeat this pass without losing provenance. + for (name, base) in &plan.candidates { + if plan.retained.contains(name) { + continue; + } + if cancelled() { + return Err(Error::Cancelled); + } + match self.store.directory().symlink_metadata(name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(_) => return Err(Error::Io), + Ok(_) => {} + } + let image = + self.store + .open_base_cancellable(name, base.length.0, &base.sha256, &cancelled)?; + gate(&cancelled, &authorize)?; + if self.store.remove_base(&image)? { + report.bases_removed += 1; + report.bytes_reclaimed.0 += base.length.0; + } + } + if plan.preparation_contract { + self.reclaim_preparations(&plan.recognized, &cancelled, &authorize, &mut report)?; + } + Ok(report) + } + fn retention_plan( + &self, + cutoff: i64, + protected: &[String], + cancelled: &impl Fn() -> bool, + ) -> Result { + let mut plan = Plan { + retire: vec![], + retained: BTreeSet::from([self.record.base.name.clone()]), + candidates: BTreeMap::new(), + stages: vec![], + recognized: BTreeSet::from([self.record.base.name.clone()]), + preparation_contract: true, + }; + if let Some(last) = &self.record.last_save { + plan.retained.insert(last.preimage.name.clone()); + plan.recognized.insert(last.preimage.name.clone()); + } + let mut seen = BTreeSet::new(); + for (count, entry) in self + .store + .directory() + .entries() + .map_err(|_| Error::Io)? + .enumerate() + { + if count >= 4096 { + return Err(Error::Limit); + } + if cancelled() { + return Err(Error::Cancelled); + } + let name = entry.map_err(|_| Error::Io)?.file_name(); + let name = name.to_str().ok_or(Error::File)?; + let Some(id) = name + .strip_suffix(".json") + .filter(|id| versions::valid_version(id)) + else { + continue; + }; + let evidence = + versions::read_any_evidence(self.store.directory(), self.store.document_id(), id)?; + if evidence.owner != self.record.owner + || evidence.target.relative != self.record.target.relative + { + return Err(Error::Changed); + } + let operation = self + .record + .operations + .iter() + .find(|operation| operation.id == evidence.info.operation_id) + .ok_or(Error::Changed)?; + if operation.outcome != Outcome::Acknowledged + || operation.file_effect != FileEffect::Changed + || operation.payload_sha256 != evidence.operation.payload_sha256 + { + return Err(Error::Changed); + } + seen.insert(operation.id.clone()); + plan.recognized + .extend([evidence.before.name.clone(), evidence.after.name.clone()]); + let keep = protected.iter().any(|value| value == id) + || operation.notification_pending + || self + .record + .last_save + .as_ref() + .is_some_and(|last| last.operation == operation.id) + || evidence.info.created_ms >= cutoff; + if evidence.retired && keep && protected.iter().any(|value| value == id) { + return Err(Error::Changed); // A checkpoint cannot silently bind expired data. + } + if !evidence.retired && keep { + plan.retained + .extend([evidence.before.name.clone(), evidence.after.name.clone()]); + } else { + if !evidence.retired { + plan.retire.push(id.into()); + } + add_candidate(&mut plan, &evidence.before)?; + add_candidate(&mut plan, &evidence.after)?; + } + if let Some(displaced) = evidence.displaced { + plan.recognized.insert(displaced.name.clone()); + plan.stages.push(Stage { + version: id.into(), + ticket: displaced, + images: [evidence.before.name.clone(), evidence.after.name.clone()], + }); + } else { + plan.preparation_contract = false; + } + } + if self.record.operations.iter().any(|operation| { + operation.outcome == Outcome::Acknowledged + && operation.file_effect == FileEffect::Changed + && !seen.contains(&operation.id) + }) { + return Err(Error::Changed); // Incomplete older evidence cannot authorize collection. + } + Ok(plan) + } + fn reclaim_preparations( + &mut self, + recognized: &BTreeSet, + cancelled: &impl Fn() -> bool, + authorize: &impl Fn() -> Result<(), Error>, + report: &mut Cleanup, + ) -> Result<(), Error> { + // Every acknowledged save has an exact stage ticket, no native intent + // remains, and the exclusive history lease excludes every prepared token. + // Other allocator-owned images therefore never reached native execution. + let mut files = vec![]; + for (count, entry) in self + .store + .directory() + .entries() + .map_err(|_| Error::Io)? + .enumerate() + { + if count >= 4096 { + return Err(Error::Limit); + } + let name = entry.map_err(|_| Error::Io)?.file_name(); + let name = name.to_str().ok_or(Error::File)?; + if !recognized.contains(name) && crate::binary::storage::preparation_name(name) { + files.push(name.to_owned()); + } + } + for name in files { + gate(cancelled, authorize)?; + if let Some(bytes) = self + .store + .remove_preparation(&name, self.record.target.metadata.uid)? + { + report.preparations_removed += 1; + report.bytes_reclaimed.0 += bytes; + } + } + Ok(()) + } + fn reclaim_stage( + &mut self, + displaced: &Displaced, + cancelled: &impl Fn() -> bool, + authorize: &impl Fn() -> Result<(), Error>, + report: &mut Cleanup, + ) -> Result<(), Error> { + if cancelled() { + return Err(Error::Cancelled); + } + match self.store.directory().symlink_metadata(&displaced.name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(_) => return Err(Error::Io), + Ok(_) => {} + } + // The immutable preimage must still exist and verify before its duplicate + // can be removed. Changed displaced data is retained as uncertain evidence. + self.store.open_base_cancellable( + &displaced.base.name, + displaced.base.length.0, + &displaced.base.sha256, + cancelled, + )?; + let verified = (|| { + let stage = stage_file(&self.store, &displaced.name)?; + if native::id(&stage)? != displaced.identity + || native::metadata(&stage)? != displaced.metadata + || stage.metadata().map_err(|_| Error::Io)?.len() != displaced.base.length.0 + || native::content(&stage, cancelled)? != displaced.base.sha256 + { + return Err(Error::Changed); + } + Ok(()) + })(); + if let Err(error) = verified { + if matches!(error, Error::Cancelled) { + return Err(error); + } + report.stages_retained += 1; + return Ok(()); + } + gate(cancelled, authorize)?; + if self + .store + .remove_known(&displaced.name, &displaced.identity)? + { + report.stages_removed += 1; + report.bytes_reclaimed.0 += displaced.base.length.0; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::binary::draft::{PatchBatch, Splice, encode_hex}; + fn save_value(editor: &mut Editor, value: u8) -> crate::binary::receipts::Version { + let id = format!("patch_{value}"); + editor + .patch( + &id, + &PatchBatch { + revision: editor.draft().revision(), + splices: vec![Splice { + offset: Offset(0), + delete_length: Offset(1), + expected_hex: encode_hex(&editor.draft().read(0, 1).unwrap()), + insert_hex: encode_hex(&[value]), + }], + }, + ) + .unwrap(); + let id = format!("save_{value}"); + let prepared = super::super::tests::ready(editor, &id); + editor.commit_save(*prepared).unwrap(); + let version = editor.publish_last_version().unwrap().unwrap(); + editor.notified(&id).unwrap(); + version + } + fn fixture() -> ( + tempfile::TempDir, + Editor, + PathBuf, + Owner, + Vec, + ) { + let (temp, mut editor, path, owner) = super::super::tests::setup(); + let versions = (1..=3) + .map(|value| save_value(&mut editor, value)) + .collect(); + (temp, editor, path, owner, versions) + } + fn cutoff() -> i64 { + chrono::Utc::now().timestamp_millis() + 1 + } + fn abandoned_preparation(editor: &mut Editor) -> (String, String) { + editor + .patch( + "abandoned", + &PatchBatch { + revision: editor.draft().revision(), + splices: vec![Splice { + offset: Offset(0), + delete_length: Offset(1), + expected_hex: "03".into(), + insert_hex: "04".into(), + }], + }, + ) + .unwrap(); + let prepared = super::super::tests::ready(editor, "abandoned_save"); + let names = ( + prepared.intent.new_base.name.clone(), + prepared.intent.stage.clone(), + ); + drop(prepared); + names + } + + #[test] + fn cleanup_preserves_dirty_current_and_last_save_images_and_is_restart_idempotent() { + let (temp, mut editor, path, owner, versions) = fixture(); + editor + .patch( + "dirty", + &PatchBatch { + revision: editor.draft().revision(), + splices: vec![Splice { + offset: Offset(1), + delete_length: Offset(1), + expected_hex: "72".into(), + insert_hex: "ff".into(), + }], + }, + ) + .unwrap(); + let dirty = editor.draft().read(0, 8).unwrap(); + let report = editor + .prune_saved_versions(cutoff(), &[], || false, || Ok(())) + .unwrap(); + assert_eq!(report.versions_retired, 2); + assert_eq!(report.bases_removed, 2); + assert_eq!(report.stages_removed, 3); + assert_eq!(editor.draft().read(0, 8).unwrap(), dirty); + assert!(editor.draft().can_undo()); + assert_eq!( + std::fs::read(&path).unwrap(), + [3, b'r', b'i', b'g', b'i', b'n', b'a', b'l'] + ); + let evidence = versions::read_evidence( + editor.store.directory(), + editor.store.document_id(), + &versions[2].version_id, + ) + .unwrap(); + evidence.before.open(&editor.store).unwrap(); + evidence.after.open(&editor.store).unwrap(); + drop(editor); + let storage = + crate::binary::storage::StorageRoot::prepare(&temp.path().join("data")).unwrap(); + let mut editor = Editor::resume(storage.open_existing("bin_one").unwrap(), &owner).unwrap(); + assert_eq!( + editor + .prune_saved_versions(cutoff(), &[], || false, || Ok(())) + .unwrap() + .bases_removed, + 0 + ); + let source = Document::open(&owner.root, "bytes").unwrap(); + let listed = storage + .versions(&source, "bin_one", None, 64, || false) + .unwrap(); + assert_eq!(listed.versions.len(), 1); + assert!( + storage + .open_version(&source, "bin_one", &versions[0].version_id, || false) + .is_err() + ); + assert_eq!( + editor.lookup_operation("save_1").unwrap().unwrap().outcome, + Outcome::Acknowledged + ); + } + #[test] + fn checkpoint_refs_and_live_prepared_tokens_prevent_history_loss() { + let (temp, mut editor, _path, owner, versions) = fixture(); + editor + .patch( + "future", + &PatchBatch { + revision: editor.draft().revision(), + splices: vec![Splice { + offset: Offset(0), + delete_length: Offset(1), + expected_hex: "03".into(), + insert_hex: "04".into(), + }], + }, + ) + .unwrap(); + let prepared = super::super::tests::ready(&mut editor, "future_save"); + assert!( + editor + .prune_saved_versions(cutoff(), &[], || false, || Ok(())) + .is_err() + ); + drop(prepared); + editor + .store + .directory() + .write("stage_unclassified", b"retain unknown evidence") + .unwrap(); + let report = editor + .prune_saved_versions( + cutoff(), + &[versions[0].version_id.clone()], + || false, + || Ok(()), + ) + .unwrap(); + assert_eq!(report.versions_retired, 1); + assert_eq!(report.bases_removed, 0); + assert_eq!(report.preparations_removed, 2); + assert_eq!( + editor.store.directory().read("stage_unclassified").unwrap(), + b"retain unknown evidence" + ); + let storage = + crate::binary::storage::StorageRoot::prepare(&temp.path().join("data")).unwrap(); + let source = Document::open(&owner.root, "bytes").unwrap(); + assert!( + storage + .open_version(&source, "bin_one", &versions[0].version_id, || false) + .is_ok() + ); + assert_eq!(editor.draft().read(0, 1).unwrap(), [4]); + } + #[test] + fn changed_displaced_data_retains_its_complete_version_and_preimages() { + let (_temp, mut editor, path, _owner, versions) = fixture(); + let first = versions::read_evidence( + editor.store.directory(), + editor.store.document_id(), + &versions[0].version_id, + ) + .unwrap(); + let displaced = first.displaced.as_ref().unwrap(); + editor + .store + .directory() + .write(&displaced.name, b"external") + .unwrap(); + let report = editor + .prune_saved_versions(cutoff(), &[], || false, || Ok(())) + .unwrap(); + assert_eq!(report.stages_retained, 1); + assert_eq!(report.versions_retired, 1); + assert_eq!(report.bases_removed, 0); + versions::read_evidence( + editor.store.directory(), + editor.store.document_id(), + &versions[0].version_id, + ) + .unwrap(); + first.before.open(&editor.store).unwrap(); + first.after.open(&editor.store).unwrap(); + assert_eq!( + editor.store.directory().read(&displaced.name).unwrap(), + b"external" + ); + assert_eq!(std::fs::read(path).unwrap()[0], 3); + } + #[test] + fn authorization_stop_and_unknown_intents_refuse_cleanup_without_source_effects() { + let (_temp, mut editor, path, _owner, _versions) = fixture(); + assert!( + editor + .prune_saved_versions(cutoff(), &[], || false, || Err(Error::Changed)) + .is_err() + ); + let stop = std::cell::Cell::new(false); + assert!(matches!( + editor.prune_saved_versions( + cutoff(), + &[], + || stop.get(), + || { + stop.set(true); + Ok(()) + } + ), + Err(Error::Cancelled) + )); + editor + .patch( + "future", + &PatchBatch { + revision: editor.draft().revision(), + splices: vec![Splice { + offset: Offset(0), + delete_length: Offset(1), + expected_hex: "03".into(), + insert_hex: "04".into(), + }], + }, + ) + .unwrap(); + let prepared = super::super::tests::ready(&mut editor, "unknown"); + editor.record.pending = Some(prepared.intent); + persist(&mut editor.store, &editor.record).unwrap(); + assert!( + editor + .prune_saved_versions(cutoff(), &[], || false, || Ok(())) + .is_err() + ); + assert!(editor.unresolved()); + assert_eq!(std::fs::read(path).unwrap()[0], 3); + } + #[test] + fn uncertain_retirement_publication_reopens_before_reclaiming_any_images() { + let (temp, mut editor, path, owner, _versions) = fixture(); + editor.store.fail_after_publication = Some(editor.store.publications + 1); + assert!( + editor + .prune_saved_versions(cutoff(), &[], || false, || Ok(())) + .is_err() + ); + assert!(editor.store.validate().is_err()); + assert_eq!(std::fs::read(&path).unwrap()[0], 3); + drop(editor); + let storage = + crate::binary::storage::StorageRoot::prepare(&temp.path().join("data")).unwrap(); + let mut editor = Editor::resume(storage.open_existing("bin_one").unwrap(), &owner).unwrap(); + let report = editor + .prune_saved_versions(cutoff(), &[], || false, || Ok(())) + .unwrap(); + assert_eq!(report.bases_removed, 2); + assert_eq!(std::fs::read(path).unwrap()[0], 3); + } + #[test] + fn incomplete_saved_evidence_cannot_authorize_collecting_preparation_files() { + let (_temp, mut editor, path, _owner, versions) = fixture(); + let (base, stage) = abandoned_preparation(&mut editor); + editor + .store + .directory() + .remove_file(format!("{}.json", versions[0].version_id)) + .unwrap(); + assert!( + editor + .prune_saved_versions(cutoff(), &[], || false, || Ok(())) + .is_err() + ); + assert!(editor.store.directory().try_exists(base).unwrap()); + assert!(editor.store.directory().try_exists(stage).unwrap()); + assert_eq!(editor.draft().read(0, 1).unwrap(), [4]); + assert_eq!(std::fs::read(path).unwrap()[0], 3); + } + #[test] + fn older_history_without_stage_tickets_keeps_unassociated_preparations() { + let (_temp, mut editor, _path, _owner, versions) = fixture(); + let mut older = versions::read_evidence( + editor.store.directory(), + editor.store.document_id(), + &versions[0].version_id, + ) + .unwrap(); + older.displaced = None; + editor + .store + .record( + &format!("{}.json", older.info.version_id), + encoded_size(&older).unwrap(), + |writer| serde_json::to_writer(writer, &older).map_err(|_| Error::Io), + ) + .unwrap(); + let (base, stage) = abandoned_preparation(&mut editor); + let protected = versions + .iter() + .map(|version| version.version_id.clone()) + .collect::>(); + let report = editor + .prune_saved_versions(cutoff(), &protected, || false, || Ok(())) + .unwrap(); + assert_eq!(report.preparations_removed, 0); + assert!(editor.store.directory().try_exists(base).unwrap()); + assert!(editor.store.directory().try_exists(stage).unwrap()); + } +} diff --git a/crates/cyber-core/src/binary/editor/versions.rs b/crates/cyber-core/src/binary/editor/versions.rs index 84e8501a..f7b958bc 100644 --- a/crates/cyber-core/src/binary/editor/versions.rs +++ b/crates/cyber-core/src/binary/editor/versions.rs @@ -10,11 +10,15 @@ pub(super) struct Evidence { pub info: Version, pub operation: Operation, pub(super) owner: Owner, - before: BaseRef, - after: BaseRef, + pub(super) before: BaseRef, + pub(super) after: BaseRef, pub target: Target, - revision: String, - ranges: Vec<(u64, u64, u64, u64)>, + pub(super) revision: String, + pub(super) ranges: Vec<(u64, u64, u64, u64)>, + #[serde(default)] + pub(super) retired: bool, + #[serde(default)] + pub(super) displaced: Option, } impl Evidence { pub fn prepared( @@ -24,6 +28,7 @@ impl Evidence { target: Target, after: &Draft, ranges: Vec<(u64, u64, u64, u64)>, + stage: &str, ) -> Result { if ranges.len() > super::super::draft::MAX_HISTORY * super::super::draft::MAX_SPLICES { return Err(Error::Limit); @@ -62,9 +67,14 @@ impl Evidence { target, revision: review.revision.clone(), ranges, + retired: false, + displaced: Some(super::retention::Displaced::prepared(editor, stage)), }) } - fn validate_common(&self, document: &str) -> Result<(), Error> { + pub(super) fn validate_common(&self, document: &str) -> Result<(), Error> { + if !valid_base(&self.before) || !valid_base(&self.after) { + return Err(Error::Changed); + } let operation = &self.operation; let expected = format!( "ver_{}", @@ -99,12 +109,19 @@ impl Evidence { || self.info.after_length != self.after.length || !self.before.name.starts_with("base_") || !self.after.name.starts_with("base_") - || self.info.range_count > self.ranges.len() + || (!self.retired && self.info.range_count > self.ranges.len()) || self.ranges.len() > crate::binary::draft::MAX_HISTORY * crate::binary::draft::MAX_SPLICES { return Err(Error::Changed); } + if self.retired { + return if self.ranges.is_empty() && self.revision.is_empty() { + Ok(()) + } else { + Err(Error::Changed) + }; + } let (mut before, mut after) = (0, 0); for &(at, removed, to, inserted) in &self.ranges { if at < before || to < after || at - before != to - after { @@ -136,10 +153,26 @@ impl Evidence { Ok(()) } } +pub(super) fn valid_base(base: &BaseRef) -> bool { + base.name.len() <= 100 + && base.name.starts_with("base_") + && base + .name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_') + && base.sha256.len() == 64 + && base.sha256.bytes().all(|b| b.is_ascii_hexdigit()) + && base.length.0 <= crate::binary::MAX_EDIT_SIZE +} impl Editor { /// Publish only after native acknowledgement is durable. The current journal /// retains the evidence if publication fails; the next save cannot discard it. pub fn publish_last_version(&mut self) -> Result, Error> { + self.store.validate()?; + let _readers = self.store.history_lease(false)?; + self.publish_last_version_locked() + } + pub(super) fn publish_last_version_locked(&mut self) -> Result, Error> { self.store.validate()?; let Some(last) = &self.record.last_save else { return Ok(None); @@ -186,8 +219,13 @@ pub struct VersionView { evidence: Evidence, before: Base, after: Base, + _readers: std::fs::File, + _source: native::Pins, } impl VersionView { + pub fn validate_resource(&self) -> Result<(), Error> { + self.evidence.target.validate_read_resource() + } pub fn info(&self) -> &Version { &self.evidence.info } @@ -197,6 +235,7 @@ impl VersionView { offset: Offset, limit: usize, ) -> Result { + self.validate_resource()?; let base = if before { &self.before } else { &self.after }; if limit == 0 || limit > super::super::MAX_PAGE || offset.0 > base.length() { return Err(Error::Limit); @@ -205,6 +244,7 @@ impl VersionView { offset.0, (base.length() - offset.0).min(limit as u64) as usize, )?; + self.validate_resource()?; Ok(super::super::Page { offset, eof: offset.0 + bytes.len() as u64 == base.length(), @@ -218,6 +258,7 @@ impl VersionView { limit: usize, cancelled: impl Fn() -> bool, ) -> Result { + self.validate_resource()?; if limit == 0 || limit > 128 { return Err(Error::Limit); } @@ -258,6 +299,7 @@ impl VersionView { { return Err(Error::Changed); } + self.validate_resource()?; Ok(Review { revision: self.evidence.revision.clone(), base_sha256: info.before_sha256.clone(), @@ -331,8 +373,11 @@ impl crate::binary::storage::StorageRoot { if after.is_some_and(|after| version <= after) { continue; } - let evidence = read_evidence(&dir, document, version)?; + let evidence = read_any_evidence(&dir, document, version)?; bind_source(&evidence, source)?; + if evidence.retired { + continue; + } found.push(evidence.info); found.sort_by(|a, b| a.version_id.cmp(&b.version_id)); found.truncate(limit + 1); @@ -368,6 +413,7 @@ impl crate::binary::storage::StorageRoot { &dir.try_clone().map_err(|_| Error::Io)?.into_std_file(), true, )?; + let readers = crate::binary::storage::history_lease(&dir, false)?; let evidence = read_evidence(&dir, document, version)?; bind_source(&evidence, source)?; let before = self.saved_base( @@ -385,10 +431,13 @@ impl crate::binary::storage::StorageRoot { &cancelled, )?; source.validate()?; + let pins = source.retain_pins()?; Ok(VersionView { evidence, before, after, + _readers: readers, + _source: pins, }) } } @@ -404,7 +453,7 @@ fn valid_document(document: &str) -> Result<(), Error> { } Ok(()) } -fn valid_version(version: &str) -> bool { +pub(super) fn valid_version(version: &str) -> bool { version.len() == 68 && version.starts_with("ver_") && version[4..].bytes().all(|b| b.is_ascii_hexdigit()) @@ -413,6 +462,17 @@ pub(super) fn read_evidence( dir: &cap_std::fs::Dir, document: &str, version: &str, +) -> Result { + let evidence = read_any_evidence(dir, document, version)?; + if evidence.retired { + return Err(Error::Changed); + } + Ok(evidence) +} +pub(super) fn read_any_evidence( + dir: &cap_std::fs::Dir, + document: &str, + version: &str, ) -> Result { let bytes = crate::binary::storage::small_record(dir, &format!("{version}.json"), 16 * 1024 * 1024)?; @@ -426,7 +486,7 @@ pub(super) fn read_evidence( fn bind_source(evidence: &Evidence, source: &Document) -> Result<(), Error> { if evidence.target.root != source.root() || evidence.target.root.join(&evidence.target.relative) != source.resource().path() - || evidence.target.parents.first() != Some(&source.project_identity()?) + || evidence.target.parents != source.parent_identities()? { return Err(Error::File); } @@ -492,6 +552,134 @@ mod tests { ); } #[test] + fn saved_views_hold_shared_leases_until_the_last_reader_closes() { + let (temp, mut editor, owner) = fixture(); + patch(&mut editor, "patch_one", &[0], &[1]); + save(&mut editor, "saved_one"); + let version = editor.publish_last_version().unwrap().unwrap(); + let storage = + crate::binary::storage::StorageRoot::prepare(&temp.path().join("data")).unwrap(); + let source = Document::open(&owner.root, "bytes").unwrap(); + let first = storage + .open_version(&source, "bin_versions", &version.version_id, || false) + .unwrap(); + let second = storage + .open_version(&source, "bin_versions", &version.version_id, || false) + .unwrap(); + assert!(editor.store.history_lease(true).is_err()); + drop(first); + assert!(editor.store.history_lease(true).is_err()); + // Ordinary source saves/publication continue while old immutable views live. + patch(&mut editor, "patch_two", &[1], &[2]); + save(&mut editor, "saved_two"); + editor.publish_last_version().unwrap(); + assert_eq!( + second.page(false, Offset(0), 4).unwrap().bytes, + [1, 255, 128, 27] + ); + drop(second); + let exclusive = editor.store.history_lease(true).unwrap(); + let source = Document::open(&owner.root, "bytes").unwrap(); + assert!( + storage + .open_version(&source, "bin_versions", &version.version_id, || false) + .is_err() + ); + drop(exclusive); + assert!( + storage + .open_version(&source, "bin_versions", &version.version_id, || false) + .is_ok() + ); + } + #[test] + fn saved_views_retain_and_check_every_source_parent_after_the_live_document_closes() { + let (temp, old, owner) = fixture(); + drop(old); + std::fs::create_dir(owner.root.join("nested")).unwrap(); + std::fs::write(owner.root.join("nested/bytes"), [0, 255, 128, 27]).unwrap(); + let storage = + crate::binary::storage::StorageRoot::prepare(&temp.path().join("data")).unwrap(); + let mut source = Document::open(&owner.root, "nested/bytes").unwrap(); + let mut editor = Editor::create( + storage.open("bin_nested").unwrap(), + owner.clone(), + &mut source, + || false, + ) + .unwrap(); + patch(&mut editor, "patch_one", &[0], &[1]); + save(&mut editor, "saved_one"); + let version = editor.publish_last_version().unwrap().unwrap(); + let source = Document::open(&owner.root, "nested/bytes").unwrap(); + let view = storage + .open_version(&source, "bin_nested", &version.version_id, || false) + .unwrap(); + drop((source, editor)); + std::fs::rename(owner.root.join("nested"), owner.root.join("original")).unwrap(); + std::fs::create_dir(owner.root.join("nested")).unwrap(); + std::fs::write(owner.root.join("nested/bytes"), [1, 255, 128, 27]).unwrap(); + let replacement = Document::open(&owner.root, "nested/bytes").unwrap(); + assert!( + storage + .open_version(&replacement, "bin_nested", &version.version_id, || false) + .is_err() + ); + assert!(view.page(false, Offset(0), 4).is_err()); + assert!(view.review(0, 1, || false).is_err()); + std::fs::remove_dir_all(owner.root.join("nested")).unwrap(); + std::fs::rename(owner.root.join("original"), owner.root.join("nested")).unwrap(); + assert_eq!( + view.page(false, Offset(0), 4).unwrap().bytes, + [1, 255, 128, 27] + ); + } + #[test] + fn retention_timestamp_is_fixed_at_commit_before_capacity_is_reserved() { + let (_temp, mut editor, _) = fixture(); + patch(&mut editor, "patch_one", &[0], &[1]); + let review = editor.review(0, 128, || false).unwrap(); + let SavePreparation::Ready(mut prepared) = editor + .prepare_save("saved", &review.fingerprint, || false) + .unwrap() + else { + panic!("save"); + }; + prepared.intent.evidence.as_mut().unwrap().info.created_ms = 0; + let before = chrono::Utc::now().timestamp_millis(); + editor.commit_save(*prepared).unwrap(); + assert!(editor.publish_last_version().unwrap().unwrap().created_ms >= before); + } + #[test] + fn a_missing_reader_lock_beside_saved_history_is_not_recreated() { + let (temp, mut editor, owner) = fixture(); + patch(&mut editor, "patch_one", &[0], &[1]); + save(&mut editor, "saved_one"); + let version = editor.publish_last_version().unwrap().unwrap(); + let storage = + crate::binary::storage::StorageRoot::prepare(&temp.path().join("data")).unwrap(); + let source = Document::open(&owner.root, "bytes").unwrap(); + let view = storage + .open_version(&source, "bin_versions", &version.version_id, || false) + .unwrap(); + let lock = storage.path().join("bin_versions/history.lock"); + std::fs::remove_file(&lock).unwrap(); + assert!(editor.store.ensure_history_lock().is_err()); + assert!(!lock.exists()); + assert!( + storage + .open_version(&source, "bin_versions", &version.version_id, || false) + .is_err() + ); + drop((editor, view)); + assert!(Editor::resume(storage.open_existing("bin_versions").unwrap(), &owner).is_err()); + assert!(!lock.exists()); + assert_eq!( + std::fs::read(owner.root.join("bytes")).unwrap(), + [1, 255, 128, 27] + ); + } + #[test] fn saved_version_listing_is_bounded_and_excludes_unsaved_history() { let (temp, mut editor, owner) = fixture(); let storage = diff --git a/crates/cyber-core/src/binary/source.rs b/crates/cyber-core/src/binary/source.rs index 41ea0be5..e7994f8c 100644 --- a/crates/cyber-core/src/binary/source.rs +++ b/crates/cyber-core/src/binary/source.rs @@ -102,8 +102,11 @@ impl Document { } #[cfg(any(target_os = "macos", target_os = "linux"))] - pub(super) fn project_identity(&self) -> Result { - super::native::id(self.directories.first().ok_or(Error::File)?.as_file()) + pub(super) fn parent_identities(&self) -> Result, Error> { + self.directories + .iter() + .map(|directory| super::native::id(directory.as_file())) + .collect() } /// Preparation requires the caller's fresh read+edit admission. It refuses diff --git a/crates/cyber-core/src/binary/storage.rs b/crates/cyber-core/src/binary/storage.rs index e550b78b..4b06ad7d 100644 --- a/crates/cyber-core/src/binary/storage.rs +++ b/crates/cyber-core/src/binary/storage.rs @@ -15,6 +15,15 @@ use std::sync::Arc; pub const MAX_STORAGE: u64 = 2 * 1024 * 1024 * 1024; const MAX_RECORD: u64 = 256 * 1024 * 1024; +#[cfg(any(target_os = "macos", target_os = "linux"))] +const HISTORY_LOCK: &str = "history.lock"; +#[cfg(any(target_os = "macos", target_os = "linux"))] +pub(super) fn preparation_name(name: &str) -> bool { + ["base_", "stage_"].iter().any(|prefix| { + name.strip_prefix(prefix) + .is_some_and(|id| id.len() == 26 && id.parse::().is_ok()) + }) +} fn open_data_directory(path: &Path) -> Result { #[cfg(windows)] @@ -440,6 +449,138 @@ fn existing_lock(dir: &Dir) -> Result { } impl Store { + #[cfg(any(target_os = "macos", target_os = "linux"))] + pub(super) fn remove_preparation( + &mut self, + name: &str, + stage_uid: u32, + ) -> Result, Error> { + self.validate()?; + if !preparation_name(name) { + return Err(Error::Invalid); + } + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + use cap_std::fs::OpenOptionsExt; + options.custom_flags(libc::O_NONBLOCK); + let file = self + .dir + .open_with(name, &options) + .map_err(|_| Error::File)? + .into_std(); + use std::os::unix::fs::MetadataExt; + let metadata = file.metadata().map_err(|_| Error::Io)?; + let owner = self + .dir + .try_clone() + .map_err(|_| Error::Io)? + .into_std_file() + .metadata() + .map_err(|_| Error::Io)? + .uid(); + let uid = if name.starts_with("stage_") { + stage_uid + } else { + owner + }; + if !metadata.is_file() + || metadata.nlink() != 1 + || metadata.uid() != uid + || metadata.len() > MAX_EDIT_SIZE + { + return Err(Error::Changed); + } + if name.starts_with("base_") { + private(&self.dir, &file, false)?; + } + let id = super::native::id(&file)?; + Ok(self.remove_known(name, &id)?.then_some(metadata.len())) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + pub(super) fn remove_known(&mut self, name: &str, expected: &str) -> Result { + component(name)?; + self.validate()?; + match self.dir.symlink_metadata(name) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(_) => return Err(Error::Io), + Ok(metadata) if !metadata.is_file() || metadata.file_type().is_symlink() => { + return Err(Error::Changed); + } + Ok(_) => {} + } + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + use cap_std::fs::OpenOptionsExt; + options.custom_flags(libc::O_NONBLOCK); + let file = self + .dir + .open_with(name, &options) + .map_err(|_| Error::Io)? + .into_std(); + use std::os::unix::fs::MetadataExt; + if file.metadata().map_err(|_| Error::Io)?.nlink() != 1 + || super::native::id(&file)? != expected + { + return Err(Error::Changed); + } + self.dir.remove_file(name).map_err(|_| Error::Io)?; + if flush_dir(&self.dir).is_err() { + self.fenced = true; + return Err(Error::Io); + } + self.validate()?; + Ok(true) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + pub(super) fn remove_base(&mut self, base: &Base) -> Result { + if generation(&base.inner.file)? != base.inner.generation { + return Err(Error::Changed); + } + let id = super::native::id(&base.inner.file)?; + self.remove_known(base.name(), &id) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + pub(super) fn ensure_history_lock(&mut self) -> Result<(), Error> { + self.validate()?; + match self.dir.symlink_metadata(HISTORY_LOCK) { + Ok(_) => { + history_lease(&self.dir, false)?; + return Ok(()); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err(Error::Io), + } + // Older private draft-only records can adopt the reader contract. Missing + // locks beside published history are never silently repaired: old readers + // may still own an earlier lock incarnation or use an older contract. + for (count, entry) in self.dir.entries().map_err(|_| Error::Io)?.enumerate() { + if count >= 4096 { + return Err(Error::Limit); + } + let name = entry.map_err(|_| Error::Io)?.file_name(); + if name + .to_str() + .is_some_and(|name| name.starts_with("ver_") && name.ends_with(".json")) + { + return Err(Error::Changed); + } + } + self.capacity(0)?; + let file = create(&self.dir, HISTORY_LOCK)?; + file.sync_all().map_err(|_| Error::Io)?; + if flush_dir(&self.dir).is_err() { + self.fenced = true; + return Err(Error::Io); + } + self.validate() + } + #[cfg(any(target_os = "macos", target_os = "linux"))] + pub(super) fn history_lease(&self, exclusive: bool) -> Result { + self.validate()?; + let lease = history_lease(&self.dir, exclusive)?; + self.validate()?; + Ok(lease) + } #[cfg(any(target_os = "macos", target_os = "linux"))] pub(super) fn storage_root(&self) -> Result { self.validate()?; @@ -759,6 +900,19 @@ impl Store { Ok(base) } } +#[cfg(any(target_os = "macos", target_os = "linux"))] +pub(super) fn history_lease(dir: &Dir, exclusive: bool) -> Result { + let file = read_file(dir, HISTORY_LOCK)?; + if file.metadata().map_err(|_| Error::Io)?.len() != 0 { + return Err(Error::Changed); + } + if exclusive { + file.try_lock().map_err(|_| Error::Limit)?; + } else { + file.try_lock_shared().map_err(|_| Error::Limit)?; + } + Ok(file) +} fn immutable_base( dir: Arc, name: &str, diff --git a/openspec/changes/add-binary-hex-editor/implementation.md b/openspec/changes/add-binary-hex-editor/implementation.md index 9bcf4f00..65dc2aef 100644 --- a/openspec/changes/add-binary-hex-editor/implementation.md +++ b/openspec/changes/add-binary-hex-editor/implementation.md @@ -497,3 +497,38 @@ passed after the post-commit lifetime changes. The independent namespace regress was split from the larger source-proof test to keep the complexity gate enabled. PR #98 merged at `69372bab` after all 22 checks passed. PR #100 carries the independent saved-version/commit-authority predecessor; the producer follow-up is not merged. + +## Retention safety work + +Saved views now retain shared private history leases. Exclusive cleanup refuses +while any view is alive, while normal saves and immutable version publication +continue. Missing reader locks beside published history are not recreated. The +lease uses the existing [Rust file locking contract](https://doc.rust-lang.org/std/fs/struct.File.html#method.try_lock_shared). +Prepared native save timestamps are fixed at the commit boundary before capacity +forecasting, so holding a prepared token does not prematurely age its history. + +Review also strengthened saved-view routing: every source parent is bound and +pinned, and page/review operations check that route before and after reading. +Replacing a source parent refuses both fresh and existing views; restoring the +original route permits the retained immutable view again. Regular source leaf +replacement by a later save remains compatible with reading prior versions. + +The binary core suite passes 72 tests locally, with the usual parent-invoked child +helper ignored in the ordinary selector. Cleanup selection, reference preservation, +crash-safe retirement/reclamation and production integration are still in progress. + +Cleanup now retires eligible acknowledged versions to compact tombstones before +reclaiming unreferenced images. Tombstones retain exact candidate references for +idempotent recovery after interrupted publication. Current/dirty bases, Last Save, +checkpoint-protected versions, pending notification evidence and live prepared save +tokens remain protected. Altered displaced files keep their complete version and +both preimages. Native intents and uncertain journals refuse cleanup. + +An exclusive history lease plus complete acknowledged-version/stage-ticket coverage +can identify allocator-owned preparation artifacts that never reached native intent; +these can be reclaimed after their token closes while preserving the durable draft. +Unclassified names, incomplete evidence and older ticketless histories are retained. +Trusted authority and cancellation are rechecked before each namespace change. +The core suite passes 79 tests, including seven cleanup regressions; core library/ +test Clippy passes. The configured seven-day policy, checkpoint reference adapter, +owner routes and operator UX still need integration before production activation.