diff --git a/AGENTS.md b/AGENTS.md index ada7970d..8be5a348 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,7 +27,7 @@ to a hot path (hashing, storage, snapshots, transfer) should come with a benchmark, and its numbers in the pull request. - `just bench`: the divan micro-benchmarks (encoding, store writes, - snapshots, input digests and promotion). + snapshots, input digests and promotion, landing). - `just bench-git`: times origin against git on a 20,000-file tree with hyperfine, and rewrites `docs/benchmarks.md`. diff --git a/README.md b/README.md index 862136bb..73cf8144 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ origin evolog # every revision of a change origin edit # switch the working copy to another change origin abandon --reason "Too slow" # give a change up, keeping it for others to see origin converge # end a change's divergence +origin resolve ... # accept conflicted files as they are origin index rebuild # rebuild the record index from the store origin annotate @ ci.test --pass --inputs src Cargo.toml # a statement about a revision @@ -31,8 +32,22 @@ origin annotate @ review --withdraw -m "Spoke too soon" # take your statement origin annotate --on flaky --neutral -m "Timeout on CI" # a statement about a statement origin statements [] # the statements about a revision, and those about them origin promote [] # carry results over from revisions with the same inputs + +origin land [] [--onto ] # land a change and the ones below it +origin land --from --onto # land a target's tip on another +origin land --status # the landings waiting for evidence or a conflict fix +origin targets # every target, its tip, authority and requirements +origin target new --from +origin new [description] --on # start a change on a target's tip ``` +Landing rebases a change onto the target's tip, file by file, checks the +statements `policy.toml` requires (promoting results whose inputs didn't +change), and appends an acceptance signed by your node, which must be the +target's authority. A landing that conflicts or lacks evidence waits: issue +the statements, or a new revision, and run `origin land` again. Policy is a +target too: edit it with `origin new --on policy`, then `origin land`. + Changes are named by a unique prefix of their id, or `@` for the working copy's. A revision is `@`, a change (for its head), or a unique prefix of the revision's id. Your name and node key live in `$ORIGIN_CONFIG_DIR`, or @@ -40,6 +55,14 @@ else `origin` in `$XDG_CONFIG_HOME` or `~/.config`: set `user = ""` in its `config.toml` if `$USER` isn't the name you want. See [the design](docs/design.md) for the model. +`converge` merges the files of a change's heads. A text file two heads +changed holds both versions between conflict markers (`<<<<<<<`, `=======`, +`>>>>>>>`), and a binary one keeps one side. A `core.conflict` statement +records which files conflict and what the merge wrote there, and the change +doesn't land while they're unchanged: edit them, or accept a file as it is +with `origin resolve `. `status` lists those left, and every new +revision, `describe` included, keeps them. + ## Development ```sh diff --git a/apps/origin-src/Cargo.toml b/apps/origin-src/Cargo.toml index 700c3eb6..fbab1ea8 100644 --- a/apps/origin-src/Cargo.toml +++ b/apps/origin-src/Cargo.toml @@ -37,6 +37,10 @@ tempfile.workspace = true name = "annotations" harness = false +[[bench]] +name = "land" +harness = false + [[bench]] name = "snapshot" harness = false diff --git a/apps/origin-src/benches/annotations.rs b/apps/origin-src/benches/annotations.rs index 16eb2ef1..a15505d8 100644 --- a/apps/origin-src/benches/annotations.rs +++ b/apps/origin-src/benches/annotations.rs @@ -180,12 +180,22 @@ fn promotions_cached(bencher: Bencher<'_, '_>) { let (_dir, store, target) = promotion_store(); let issuer: Principal = "node:queue".parse().unwrap(); let mut annotations = Annotations::open(&store).unwrap(); - assert_eq!(annotations.promotions(target, &issuer).unwrap().len(), 9); + assert_eq!( + annotations + .promotions(RecordId::Revision(target), &issuer) + .unwrap() + .len(), + 9 + ); let index = Index::open(&store).unwrap(); bencher .counter(ItemsCount::new(STATEMENTS)) .with_inputs(|| Annotations::new(&store, index.clone())) - .bench_local_refs(|annotations| annotations.promotions(target, &issuer).unwrap()); + .bench_local_refs(|annotations| { + annotations + .promotions(RecordId::Revision(target), &issuer) + .unwrap() + }); } /// The first search, which computes and remembers the digest of each of @@ -202,5 +212,9 @@ fn promotions_uncached(bencher: Bencher<'_, '_>) { fs::remove_file(&log).unwrap(); Annotations::open(&store).unwrap() }) - .bench_local_refs(|annotations| annotations.promotions(target, &issuer).unwrap()); + .bench_local_refs(|annotations| { + annotations + .promotions(RecordId::Revision(target), &issuer) + .unwrap() + }); } diff --git a/apps/origin-src/benches/land.rs b/apps/origin-src/benches/land.rs new file mode 100644 index 00000000..8a152d3e --- /dev/null +++ b/apps/origin-src/benches/land.rs @@ -0,0 +1,235 @@ +//! The landing queue: the latency of one landing, made on the target's tip +//! or rebased onto it, with and without a requirement, of a target landing +//! another after long histories, and the throughput of several proposers +//! landing different changes on one target at once. Run on a real disk: +//! syncs are most of the cost. + +use std::thread; + +use divan::Bencher; +use divan::counter::ItemsCount; +use origin_platform::Native; +use origin_shared::{ + AcceptanceId, Base, ChangeId, EncodedObject, EntryKind, EntryName, Kind, NodeKey, + PrincipalName, Record, RecordId, Revision, Statement, TargetName, Tree, TreeEntry, Verdict, +}; +use origin_src::annotations::Annotations; +use origin_src::queue::{Landing, Queue}; +use origin_src::store::Store; + +fn main() { + divan::main(); +} + +/// A repository whose `policy` and `main` the node `queue` orders. +struct Repo { + _dir: tempfile::TempDir, + store: Store, + key: NodeKey, +} + +impl Repo { + /// A repository where landing on `main` requires `require`, a TOML + /// array. + fn new(require: &str) -> Self { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let repo = Self { + _dir: dir, + store, + key: NodeKey::from_bytes(&[7; 32]), + }; + let policy = format!( + "[nodes]\nqueue = \"{}\"\n[groups]\nrunners = [\"node:ci\"]\n\ + [targets.policy]\nauthority = \"node:queue\"\n\ + [targets.feature]\nauthority = \"node:queue\"\n\ + [targets.main]\nauthority = \"node:queue\"\nrequire = {require}\n", + repo.key.public() + ); + repo.queue().init(&policy).unwrap(); + repo + } + + fn queue(&self) -> Queue<'_> { + Queue::new(&self.store, &self.key, PrincipalName::new("queue").unwrap()) + } + + fn tip(&self) -> AcceptanceId { + self.tip_of(&TargetName::MAIN) + } + + fn tip_of(&self, target: &TargetName) -> AcceptanceId { + self.store.head(target).unwrap().unwrap().acceptance + } + + /// Writes a change made on `base`, a tip of `main`, adding the file + /// `file-`. + fn change(&self, n: usize, base: AcceptanceId) -> ChangeId { + let mut tree: Tree = self + .store + .get(self.store.get(base).unwrap().body().state) + .unwrap(); + let content = EncodedObject::blob(format!("fn f{n}() {{}}\n").as_bytes()); + tree.insert( + EntryName::new(format!("file-{n}")).unwrap(), + TreeEntry { + kind: EntryKind::File, + id: content.id(), + }, + ); + let tree = tree.encode(); + let mut id = [0; 16]; + id[..8].copy_from_slice(&(n as u64).to_le_bytes()); + let change = ChangeId::from_bytes(id); + let revision = Revision { + change, + predecessors: Vec::new(), + base: Base::Acceptance(base), + deps: Vec::new(), + snapshot: tree.id().into(), + description: EncodedObject::blob(b"").id().into(), + author: "user:bench".parse().unwrap(), + } + .encode(); + let description = EncodedObject::blob(b""); + self.store.write_all(&[content, tree, description]).unwrap(); + self.store.write_all(&[revision]).unwrap(); + change + } + + fn land(&self, change: ChangeId) { + self.land_on(change, &TargetName::MAIN); + } + + fn land_on(&self, change: ChangeId, onto: &TargetName) { + let landings = self.queue().land_change(change, onto).unwrap(); + assert!( + matches!(landings[..], [Landing::Landed { .. }]), + "{landings:?}" + ); + } +} + +/// One landing of a change made on the tip: no rebase, no requirement. +#[divan::bench(sample_count = 50)] +fn land_on_tip(bencher: Bencher<'_, '_>) { + let repo = Repo::new("[]"); + let mut n = 0; + bencher + .with_inputs(|| { + n += 1; + repo.change(n, repo.tip()) + }) + .bench_local_values(|change| repo.land(change)); +} + +/// One landing of a change made on an older tip, which the queue rebases +/// into a new revision. +#[divan::bench(sample_count = 50)] +fn land_rebased(bencher: Bencher<'_, '_>) { + let repo = Repo::new("[]"); + let first = repo.tip(); + let mut n = 0; + bencher + .with_inputs(|| { + n += 1; + repo.change(n, first) + }) + .bench_local_values(|change| repo.land(change)); +} + +/// One landing of a change made on the tip, with the `ci.test` result +/// `main` requires. +#[divan::bench(sample_count = 50)] +fn land_with_requirement(bencher: Bencher<'_, '_>) { + let repo = Repo::new(r#"[{ kind = "ci.test", verdict = "pass", issuer = "group:runners" }]"#); + let mut n = 0; + bencher + .with_inputs(|| { + n += 1; + let change = repo.change(n, repo.tip()); + let annotations = Annotations::open(&repo.store).unwrap(); + let head = annotations.index().heads(change)[0]; + let result = Statement { + subject: RecordId::Revision(head), + kind: Kind::new("ci.test").unwrap(), + verdict: Some(Verdict::Pass), + issuer: "node:ci".parse().unwrap(), + inputs: None, + payload: None, + supersedes: None, + promoted_from: None, + }; + annotations.issue(&result, None).unwrap(); + change + }) + .bench_local_values(|change| repo.land(change)); +} + +/// How many changes each target lands before they merge. +const HISTORY: usize = 1_000; + +/// Landing the tip of `feature` on `main` once each landed `HISTORY` +/// changes since `feature` forked, after `feature` lands one more: the +/// merge base reads both histories whole. +#[divan::bench(sample_count = 20, sample_size = 1)] +fn land_target_after_long_histories(bencher: Bencher<'_, '_>) { + let repo = Repo::new("[]"); + let feature = TargetName::new("feature").unwrap(); + repo.queue() + .create_target(&feature, &TargetName::MAIN) + .unwrap(); + for n in 0..HISTORY { + repo.land(repo.change(n, repo.tip())); + let change = repo.change(HISTORY + n, repo.tip_of(&feature)); + repo.land_on(change, &feature); + } + let mut n = 2 * HISTORY; + bencher + .with_inputs(|| { + n += 1; + let change = repo.change(n, repo.tip_of(&feature)); + repo.land_on(change, &feature); + }) + .bench_local_values(|()| { + let landing = repo + .queue() + .land_target(&feature, &TargetName::MAIN) + .unwrap(); + assert!(matches!(landing, Landing::Landed { .. }), "{landing:?}"); + }); +} + +/// How many changes each proposer lands. +const EACH: usize = 8; + +/// `proposers` threads landing `EACH` changes each on `main` at once, all +/// made on its first tip, so all but the first are rebased. Threads lock +/// the queue as processes do: the lock belongs to the open file. +#[divan::bench(args = [1, 2, 4, 8], sample_count = 5, sample_size = 1)] +fn concurrent_proposers(bencher: Bencher<'_, '_>, proposers: usize) { + bencher + .counter(ItemsCount::new(proposers * EACH)) + .with_inputs(|| { + let repo = Repo::new("[]"); + let first = repo.tip(); + let changes: Vec<_> = (0..proposers * EACH) + .map(|n| repo.change(n, first)) + .collect(); + (repo, changes) + }) + .bench_local_values(|(repo, changes)| { + thread::scope(|scope| { + for changes in changes.chunks(EACH) { + let repo = &repo; + scope.spawn(move || { + for &change in changes { + repo.land(change); + } + }); + } + }); + (repo, changes) + }); +} diff --git a/apps/origin-src/src/ancestry.rs b/apps/origin-src/src/ancestry.rs new file mode 100644 index 00000000..8278e72f --- /dev/null +++ b/apps/origin-src/src/ancestry.rs @@ -0,0 +1,195 @@ +//! Merge bases in a history of immutable records with snapshots: the +//! acceptances of targets, each after the one before it and the +//! acceptances it landed, or the revisions of a change, each after its +//! predecessors. + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; + +use origin_platform::Platform; +use origin_shared::TreeId; + +use crate::store::{Store, StoreError}; + +/// A history, as far as it was walked: the parents and the snapshot of +/// each record in it. +#[derive(Debug)] +pub struct Ancestry { + records: HashMap, TreeId)>, +} + +impl Ancestry { + /// Walks everything `tips` descend from, asking `read` for the parents + /// and the snapshot of each record, once. + /// + /// # Errors + /// + /// Fails as `read` does. + pub fn walk( + tips: &[I], + mut read: impl FnMut(I) -> Result<(Vec, TreeId), E>, + ) -> Result { + let mut records = HashMap::new(); + let mut next = tips.to_vec(); + while let Some(id) = next.pop() { + if records.contains_key(&id) { + continue; + } + let record = read(id)?; + next.extend(&record.0); + records.insert(id, record); + } + Ok(Self { records }) + } + + /// The snapshot to merge `tips` from, `None` if they share no ancestor: + /// that of their best common ancestor. Several, as histories that + /// merged each other criss-cross have, merge into one first, as git's + /// recursive strategy does: each from their own merge base, with what + /// [`Store::merge_bases`] leaves where they conflict, so a merge from + /// the result conflicts there, and under it, unless both sides agree. Writes the trees + /// that makes, which merging reads. + /// + /// # Errors + /// + /// Fails if a tree cannot be read or written. + pub fn merge_base( + &self, + store: &Store

, + tips: &[I], + ) -> Result, StoreError> { + let groups: Vec<_> = tips.iter().map(std::slice::from_ref).collect(); + self.merge_base_of(store, &groups) + } + + /// The snapshot to merge `groups` from, each standing for all it holds. + fn merge_base_of( + &self, + store: &Store

, + groups: &[&[I]], + ) -> Result, StoreError> { + let bases = self.best_common(groups); + let Some((&first, others)) = bases.split_first() else { + return Ok(None); + }; + let mut merged = vec![first]; + let mut tree = self.snapshot(first); + for &other in others { + let base = self.merge_base_of(store, &[&merged, &[other]])?; + let merge = store.merge_bases(base, tree, self.snapshot(other))?; + store.write_all(&merge.trees)?; + tree = merge.tree; + merged.push(other); + } + Ok(Some(tree)) + } + + /// The best common ancestors of `groups`: the records that every group + /// has one that is or descends from, but that none of the others + /// descends from, sorted. Each record is visited a bounded number of + /// times. + fn best_common(&self, groups: &[&[I]]) -> Vec { + let Some((first, others)) = groups.split_first() else { + return Vec::new(); + }; + let mut common = self.reachable(first.iter().copied()); + for group in others { + let reachable = self.reachable(group.iter().copied()); + common.retain(|id| reachable.contains(id)); + } + let parents = common.iter().flat_map(|&id| self.parents(id)); + let below = self.reachable(parents.copied()); + let mut bases: Vec<_> = common.difference(&below).copied().collect(); + bases.sort_unstable(); + bases + } + + /// The records `from` are or descend from. + fn reachable(&self, from: impl IntoIterator) -> HashSet { + let mut reached = HashSet::new(); + let mut next: Vec<_> = from.into_iter().collect(); + while let Some(id) = next.pop() { + if reached.insert(id) { + next.extend(self.parents(id)); + } + } + reached + } + + /// Whether `id` is or descends from `ancestor`, as far as the walk + /// read. + #[must_use] + pub fn descends(&self, id: I, ancestor: I) -> bool { + self.reachable([id]).contains(&ancestor) + } + + fn parents(&self, id: I) -> &[I] { + self.records.get(&id).map_or(&[], |(parents, _)| parents) + } + + fn snapshot(&self, id: I) -> TreeId { + self.records[&id].1 + } +} + +#[cfg(test)] +mod tests { + use origin_shared::{Record, Tree}; + + use super::*; + + /// The history of `edges`, each a record and its parents. + fn ancestry(edges: &[(u8, &[u8])]) -> Ancestry { + let tips: Vec<_> = edges.iter().map(|&(id, _)| id).collect(); + let empty = Tree::default().encode().id().into(); + Ancestry::walk(&tips, |id| { + let parents = edges.iter().find(|&&(of, _)| of == id).map(|&(_, p)| p); + Ok::<_, ()>((parents.unwrap_or_default().to_vec(), empty)) + }) + .unwrap() + } + + fn bases(history: &Ancestry, tips: &[u8]) -> Vec { + let groups: Vec<_> = tips.iter().map(std::slice::from_ref).collect(); + history.best_common(&groups) + } + + #[test] + fn the_newest_shared_ancestor_wins_however_far() { + // 2 follows 1; 3 to 6 and 7 to 10 grow apart from 2, then each + // lands a side line forked from 1: 1 is nearer, but 2 is newer. + let history = ancestry(&[ + (2, &[1]), + (3, &[2]), + (4, &[3]), + (5, &[4]), + (6, &[5]), + (7, &[2]), + (8, &[7]), + (9, &[8]), + (10, &[9]), + (11, &[1]), + (12, &[6, 11]), + (13, &[1]), + (14, &[10, 13]), + ]); + assert_eq!(bases(&history, &[12, 14]), [2]); + assert_eq!(bases(&history, &[12, 6]), [6], "an ancestor of the other"); + } + + #[test] + fn criss_cross_merges_have_two_bases() { + // 2 and 3 fork from 1, then each merges the other. + let history = ancestry(&[(2, &[1]), (3, &[1]), (4, &[2, 3]), (5, &[3, 2])]); + assert_eq!(bases(&history, &[4, 5]), [2, 3]); + assert_eq!(bases(&history, &[4, 5, 3]), [3]); + // Merging those two needs their own base. + assert_eq!(history.best_common(&[&[2], &[3]]), [1]); + } + + #[test] + fn unrelated_histories_have_none() { + let history = ancestry(&[(2, &[1]), (4, &[3])]); + assert_eq!(bases(&history, &[2, 4]), []); + } +} diff --git a/apps/origin-src/src/annotations/mod.rs b/apps/origin-src/src/annotations/mod.rs index 60da8af6..a82ce0ee 100644 --- a/apps/origin-src/src/annotations/mod.rs +++ b/apps/origin-src/src/annotations/mod.rs @@ -300,7 +300,9 @@ pub enum AnnotateError { #[cfg(test)] mod tests { - use origin_shared::{Base, BlobId, Digest, Inputs, ObjectId, RepoPath, Revision, Verdict}; + use origin_shared::{ + Base, BlobId, Digest, Inputs, ObjectId, RepoPath, Revision, RevisionId, Verdict, + }; use super::*; use crate::tree::tests::build; @@ -382,7 +384,7 @@ mod tests { let promoter = principal("node:queue"); Annotations::open(store) .unwrap() - .promote(revision, &promoter) + .promote(RecordId::Revision(revision), &promoter) .unwrap() .into_iter() .map(|(_, statement)| { diff --git a/apps/origin-src/src/annotations/promotion.rs b/apps/origin-src/src/annotations/promotion.rs index abef909a..c2610504 100644 --- a/apps/origin-src/src/annotations/promotion.rs +++ b/apps/origin-src/src/annotations/promotion.rs @@ -17,7 +17,7 @@ use std::collections::HashSet; use origin_platform::Platform; -use origin_shared::{Principal, Record, RecordId, RevisionId, Statement, StatementId, TreeId}; +use origin_shared::{Principal, Record, RecordId, Statement, StatementId, TreeId}; use super::Annotations; use crate::store::{Store, StoreError}; @@ -140,13 +140,14 @@ impl Annotations<'_, P> { self.index.remember_digests(self.store, &digests.concat()) } - /// The statements promoting results to `revision`, as `issuer` would + /// The statements promoting results to `subject`, as `issuer` would /// issue them, without writing them, sorted by kind: for each /// [promotion group](Statement::promotion_group), the latest statement - /// about another record with the same input digest as the revision, + /// about another record with the same input digest as the subject, /// unless its issuer superseded it. Groups whose kind and issuer the - /// revision already has a result of, directly or promoted, are skipped: - /// a result that ran there wins. + /// subject already has a result of, directly or promoted, are skipped: + /// a result that ran there wins. A statement has no snapshot, so + /// nothing is promoted to one. /// /// # Errors /// @@ -154,11 +155,12 @@ impl Annotations<'_, P> { /// be remembered. pub fn promotions( &mut self, - revision: RevisionId, + subject: RecordId, issuer: &Principal, ) -> Result, StoreError> { - let subject = RecordId::Revision(revision); - let snapshot = self.store.get(revision)?.snapshot; + let Some(snapshot) = self.store.snapshot_of(subject)? else { + return Ok(Vec::new()); + }; self.compute_digests()?; // The results the revision has that hold, by kind and original @@ -203,7 +205,7 @@ impl Annotations<'_, P> { Ok(promotions) } - /// Issues the [`promotions`](Self::promotions) to `revision` as + /// Issues the [`promotions`](Self::promotions) to `subject` as /// `issuer`, returning them. /// /// # Errors @@ -212,10 +214,10 @@ impl Annotations<'_, P> { /// be written. pub fn promote( &mut self, - revision: RevisionId, + subject: RecordId, issuer: &Principal, ) -> Result, StoreError> { - let promotions = self.promotions(revision, issuer)?; + let promotions = self.promotions(subject, issuer)?; let encoded: Vec<_> = promotions.iter().map(Statement::encode).collect(); self.store.write_all(&encoded)?; Ok(encoded diff --git a/apps/origin-src/src/cli.rs b/apps/origin-src/src/cli.rs index d7cd1b76..b80df439 100644 --- a/apps/origin-src/src/cli.rs +++ b/apps/origin-src/src/cli.rs @@ -53,6 +53,9 @@ enum WorkspaceCommand { /// What the new change is for. #[arg(default_value = "")] description: String, + /// Start it on the tip of this target instead. + #[arg(long)] + on: Option, }, /// Describe what the current change is for. Describe { @@ -78,6 +81,12 @@ enum WorkspaceCommand { /// The change. change: String, }, + /// Accept conflicted files as they are, resolving their conflicts. + Resolve { + /// The files, from the root of the repository. + #[arg(required = true)] + paths: Vec, + }, /// Show the working copy's changes and the target they're on. Log { /// Show this target's history instead. @@ -119,6 +128,44 @@ enum WorkspaceCommand { #[arg(default_value = "@")] rev: String, }, + /// Land a change on a target, with the changes below it the target + /// doesn't have yet. The queue rebases it onto the target's tip, checks + /// the evidence `policy.toml` requires, and if it's there, appends a + /// signed acceptance. A landing that conflicts or lacks evidence waits. + Land(LandArgs), + /// Manage targets. + #[command(subcommand)] + Target(TargetCommand), + /// List the targets, with their tips, authorities and requirements. + Targets, +} + +#[derive(Debug, Args)] +struct LandArgs { + /// The change: the working copy's by default. + #[arg(conflicts_with_all = ["from", "status"])] + change: Option, + /// The target to land on: by default, the one the change is on. + #[arg(long)] + onto: Option, + /// Land the tip of this target instead of a change. + #[arg(long, requires = "onto")] + from: Option, + /// List the landings that wait, and why. + #[arg(long, conflicts_with_all = ["onto", "from"])] + status: bool, +} + +#[derive(Debug, Subcommand)] +enum TargetCommand { + /// Create a target at the tip of another. + New { + /// Its name, such as `release/v3`. + name: TargetName, + /// The target it starts from. + #[arg(long)] + from: TargetName, + }, } #[derive(Debug, Args)] @@ -331,8 +378,12 @@ impl WorkspaceCommand { Self::Status => workspace.status()?, Self::Diff => workspace.diff()?, Self::Show { change } => workspace.show(change.as_deref())?, - Self::New { description } => workspace.new_change(&description)?, + Self::New { description, on } => match on { + Some(target) => workspace.new_on(&target, &description)?, + None => workspace.new_change(&description)?, + }, Self::Describe { description } => workspace.describe(&description)?, + Self::Resolve { paths } => workspace.resolve(&paths)?, Self::Edit { change } => workspace.edit(&change)?, Self::Abandon { change, reason } => workspace.abandon(change.as_deref(), &reason)?, Self::Converge { change } => workspace.converge(&change)?, @@ -351,6 +402,15 @@ impl WorkspaceCommand { } Self::Statements { rev } => workspace.statements(&rev)?, Self::Promote { rev } => workspace.promote(&rev)?, + Self::Land(args) => match (args.from, args.onto) { + _ if args.status => workspace.land_status()?, + (Some(from), Some(onto)) => workspace.land_target(&from, &onto)?, + (_, onto) => workspace.land(args.change.as_deref(), onto)?, + }, + Self::Target(TargetCommand::New { name, from }) => { + workspace.target_new(&name, &from)?; + } + Self::Targets => workspace.targets()?, } Ok(()) } diff --git a/apps/origin-src/src/conflict.rs b/apps/origin-src/src/conflict.rs new file mode 100644 index 00000000..4367cad4 --- /dev/null +++ b/apps/origin-src/src/conflict.rs @@ -0,0 +1,416 @@ +//! Conflicts: what a merge leaves where both sides changed a file. +//! +//! A merge into a working copy, such as `origin converge`, writes both +//! versions of each such text file, between markers naming the sides: +//! +//! ```text +//! <<<<<<< 3f1c9a2e07b4 +//! our version +//! ======= +//! their version +//! >>>>>>> 8d02e5b1c4f9 +//! ``` +//! +//! A binary file, or a path a directory is on either side of, keeps one +//! side instead. The merge then issues a `core.conflict` statement about +//! the new revision, as its author, a user, listing each conflicted path +//! and the entry it wrote there. That list is the conflict, not the +//! markers, which only help to edit the file: a path stays unresolved while +//! a revision has the entry written there. Changing the file resolves it, +//! and so does `origin resolve`, which accepts it as it is, such as the +//! side a binary file kept. +//! +//! Every revision made from one with unresolved paths, by a snapshot, +//! `describe`, a new change on top, or a merge, carries those it still has +//! the entries of over, in a `core.conflict` of its own, published before +//! the revision itself, so it's never found without it. The landing queue +//! refuses a revision whose author's latest `core.conflict` lists a path it +//! still has the entry of. +//! +//! The landing queue also issues `core.conflict`, as a node, when a revision +//! doesn't rebase onto its target, even one the queue rebased itself. That +//! one only blocks the landing on that target, which merges again whenever +//! it's retried. + +use origin_platform::Platform; +use origin_shared::{ + EncodedObject, Kind, ObjectId, Principal, RecordId, RepoPath, Revision, Statement, StatementId, + TreeId, +}; + +use crate::annotations::Claim; +use crate::store::{Store, StoreError}; +use crate::tree::Snapshots; + +/// A path a merge left conflicted, and the entry it wrote there. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Conflicted { + /// The path. + pub path: RepoPath, + /// The id of the entry written there: both versions between markers, + /// or the side kept. `None` if the merge left nothing there. + pub written: Option, +} + +/// Conflicted paths, sorted, each at most once: those a merge left, or a +/// revision's unresolved ones. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Conflicts(Vec); + +impl Conflicts { + /// The line starting our side, then its label. + const OURS: &str = "<<<<<<< "; + /// The line between the sides. + const BETWEEN: &str = "======="; + /// The line ending their side, then its label. + const THEIRS: &str = ">>>>>>> "; + + /// The conflicts of `conflicted`, the latest of each path counting. + #[must_use] + pub fn new(mut conflicted: Vec) -> Self { + // Stable, so the latest of each path comes last, and dedup keeps the + // first of equal ones: reverse around it. + conflicted.reverse(); + conflicted.sort_by(|a, b| a.path.cmp(&b.path)); + conflicted.dedup_by(|a, b| a.path == b.path); + Self(conflicted) + } + + /// Adds `conflicted`, which must come after every path listed. + pub(crate) fn push(&mut self, conflicted: Conflicted) { + debug_assert!(self.0.last().is_none_or(|last| last.path < conflicted.path)); + self.0.push(conflicted); + } + + /// Whether there are none. + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// The conflicted paths, sorted. + #[must_use] + pub fn paths(&self) -> Vec { + self.0 + .iter() + .map(|conflicted| conflicted.path.clone()) + .collect() + } + + /// These conflicts, then `newer` ones, which replace those of the same + /// paths: what a merge of a revision with these conflicts leaves. + #[must_use] + pub fn with(mut self, newer: Self) -> Self { + self.0.extend(newer.0); + Self::new(self.0) + } + + /// Resolves the conflicts of `paths`, returning those it had. + pub fn resolve(&mut self, paths: &[RepoPath]) -> Vec { + let mut resolved = Vec::new(); + self.0.retain(|conflicted| { + let listed = paths.contains(&conflicted.path); + if listed { + resolved.push(conflicted.path.clone()); + } + !listed + }); + resolved + } + + /// The content of a file both sides changed: `ours`, then `theirs`, + /// between markers naming them with `labels`. A side that removed the + /// file is empty. `None` if either side is binary, holding a NUL byte: + /// markers would only corrupt it. + #[must_use] + pub fn mark(ours: &[u8], theirs: &[u8], labels: [&str; 2]) -> Option> { + if ours.contains(&0) || theirs.contains(&0) { + return None; + } + let mut marked = Vec::with_capacity(ours.len() + theirs.len() + 64); + let side = |marked: &mut Vec, content: &[u8]| { + marked.extend_from_slice(content); + if !content.is_empty() && !content.ends_with(b"\n") { + marked.push(b'\n'); + } + }; + marked.extend_from_slice(format!("{}{}\n", Self::OURS, labels[0]).as_bytes()); + side(&mut marked, ours); + marked.extend_from_slice(format!("{}\n", Self::BETWEEN).as_bytes()); + side(&mut marked, theirs); + marked.extend_from_slice(format!("{}{}\n", Self::THEIRS, labels[1]).as_bytes()); + Some(marked) + } + + /// The `core.conflict` statement about `subject`, from `issuer`, + /// listing these conflicts in its payload, and the payload: each one + /// as the id written, or `-` for none, a space and the path, ended by a + /// NUL byte, which no path holds. + #[must_use] + pub fn statement( + &self, + subject: RecordId, + issuer: Principal, + supersedes: Option, + ) -> (EncodedObject, Statement) { + let mut list = String::new(); + for conflicted in &self.0 { + match conflicted.written { + Some(id) => list.push_str(&id.to_string()), + None => list.push('-'), + } + list.push(' '); + list.push_str(conflicted.path.as_str()); + list.push('\0'); + } + let payload = EncodedObject::blob(list.as_bytes()); + let statement = Statement { + subject, + kind: Kind::CONFLICT, + verdict: None, + issuer, + inputs: None, + payload: Some(payload.id().into()), + supersedes, + promoted_from: None, + }; + (payload, statement) + } + + /// The conflicts a `core.conflict` statement lists. Its payload is the + /// list, which a repository always keeps, so one it lacks fails rather + /// than reading as no conflicts. + /// + /// # Errors + /// + /// Fails if the payload is missing or cannot be read. + pub fn listed( + store: &Store

, + statement: &Statement, + ) -> Result { + let Some(payload) = statement.payload else { + return Ok(Self::default()); + }; + let content = store + .try_blob(payload.into())? + .ok_or(StoreError::NotFound(payload.into()))?; + let conflicted = String::from_utf8_lossy(&content) + .split_terminator('\0') + .filter_map(|line| { + let (written, path) = line.split_once(' ')?; + let written = match written { + "-" => None, + id => Some(id.parse().ok()?), + }; + let path = RepoPath::new(path).ok()?; + Some(Conflicted { path, written }) + }) + .collect(); + Ok(Self::new(conflicted)) + } + + /// The unresolved conflicts of `revision`, from `claims`, those about it + /// that hold: of the latest `core.conflict` from its author, if that's + /// a user, the paths where it still has the entry written. A node's + /// says only that a landing queue could not rebase it. + /// + /// # Errors + /// + /// Fails if the statement's payload is missing, or it or a tree on the + /// way to a path cannot be read. + pub fn of( + store: &Store

, + claims: &[Claim], + revision: &Revision, + ) -> Result { + if matches!(revision.author, Principal::Node(_)) { + return Ok(Self::default()); + } + let Some(claim) = claims.iter().rev().find(|claim| { + claim.statement().kind == Kind::CONFLICT && claim.statement().issuer == revision.author + }) else { + return Ok(Self::default()); + }; + Self::listed(store, claim.statement())?.remaining(store, revision.snapshot) + } + + /// Those of these conflicts `snapshot` still has: where it has the + /// entry written. + /// + /// # Errors + /// + /// Fails if a tree on the way to a path cannot be read. + pub fn remaining( + mut self, + store: &Store

, + snapshot: TreeId, + ) -> Result { + let mut snapshots = Snapshots::new(store); + let mut remaining = Vec::with_capacity(self.0.len()); + for conflicted in self.0.drain(..) { + let entry = snapshots.entry_at(snapshot, &conflicted.path)?; + if entry.map(|entry| entry.id) == conflicted.written { + remaining.push(conflicted); + } + } + Ok(Self(remaining)) + } +} + +#[cfg(test)] +mod tests { + use origin_platform::Native; + use origin_shared::{EntryKind, EntryName, Record, Tree, TreeEntry}; + + use super::*; + + fn path(path: &str) -> RepoPath { + RepoPath::new(path).unwrap() + } + + #[test] + fn a_path_resolves_once_its_entry_changes() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let marked = EncodedObject::blob(b"<<<<<<< a\nours\n=======\ntheirs\n>>>>>>> b\n"); + let resolved = EncodedObject::blob(b"Heading\n=======\nresolved\n"); + let snapshot = |file: &EncodedObject| { + let mut tree = Tree::default(); + tree.insert( + EntryName::new("file").unwrap(), + TreeEntry { + kind: EntryKind::File, + id: file.id(), + }, + ); + let tree = tree.encode(); + store.write_all(&[file.clone(), tree.clone()]).unwrap(); + TreeId::from(tree.id()) + }; + let conflicts = Conflicts::new(vec![ + Conflicted { + path: path("file"), + written: Some(marked.id()), + }, + Conflicted { + path: path("removed"), + written: None, + }, + ]); + let (payload, statement) = conflicts.statement( + RecordId::Snapshot(snapshot(&marked)), + "user:ada".parse().unwrap(), + None, + ); + // Without its payload, the statement lists nothing it can trust. + assert!(matches!( + Conflicts::listed(&store, &statement), + Err(StoreError::NotFound(id)) if id == payload.id() + )); + store.write_all(&[payload]).unwrap(); + assert_eq!(Conflicts::listed(&store, &statement).unwrap(), conflicts); + + let unchanged = conflicts.clone().remaining(&store, snapshot(&marked)); + assert_eq!(unchanged.unwrap(), conflicts); + // Even to content that looks like a marker. + let remaining = conflicts.remaining(&store, snapshot(&resolved)).unwrap(); + assert_eq!(remaining.paths(), [path("removed")]); + } + + #[test] + fn only_a_user_author_leaves_a_revision_unresolved() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let tree = Tree::default().encode(); + store.write_all(std::slice::from_ref(&tree)).unwrap(); + let revision = |author: &str| Revision { + change: origin_shared::ChangeId::from_bytes([1; 16]), + predecessors: Vec::new(), + base: origin_shared::Base::Acceptance(EncodedObject::blob(b"base").id().into()), + deps: Vec::new(), + snapshot: tree.id().into(), + description: EncodedObject::blob(b"").id().into(), + author: author.parse().unwrap(), + }; + let subject = RecordId::Revision(revision("user:ada").encode().id().into()); + let conflicts = Conflicts::new(vec![Conflicted { + path: path("file"), + written: None, + }]); + let claims = |issuer: &str| { + let (payload, statement) = conflicts.statement(subject, issuer.parse().unwrap(), None); + store.write_all(&[payload]).unwrap(); + let id = statement.encode().id().into(); + [store.claim(id, statement).unwrap().unwrap()] + }; + let ada = revision("user:ada"); + assert_eq!( + Conflicts::of(&store, &claims("user:ada"), &ada).unwrap(), + conflicts + ); + assert!( + Conflicts::of(&store, &claims("user:eve"), &ada) + .unwrap() + .is_empty() + ); + // A queue that rebased a revision, then could not rebase it elsewhere. + let queue = revision("node:queue"); + assert!( + Conflicts::of(&store, &claims("node:queue"), &queue) + .unwrap() + .is_empty() + ); + } + + #[test] + fn newer_conflicts_replace_older_ones() { + let older = Conflicts::new(vec![ + Conflicted { + path: path("a"), + written: None, + }, + Conflicted { + path: path("b"), + written: None, + }, + ]); + let id = EncodedObject::blob(b"newer").id(); + let newer = Conflicts::new(vec![Conflicted { + path: path("b"), + written: Some(id), + }]); + let mut both = older.with(newer); + assert_eq!( + both, + Conflicts::new(vec![ + Conflicted { + path: path("a"), + written: None, + }, + Conflicted { + path: path("b"), + written: Some(id), + }, + ]) + ); + assert_eq!(both.resolve(&[path("a"), path("c")]), [path("a")]); + assert_eq!(both.paths(), [path("b")]); + } + + #[test] + fn marks_both_sides() { + let marked = Conflicts::mark(b"ours\n", b"theirs", ["a", "b"]).unwrap(); + assert_eq!( + String::from_utf8(marked).unwrap(), + "<<<<<<< a\nours\n=======\ntheirs\n>>>>>>> b\n" + ); + assert_eq!( + Conflicts::mark(b"", b"kept\n", ["a", "b"]).unwrap(), + b"<<<<<<< a\n=======\nkept\n>>>>>>> b\n", + "a removed side is empty" + ); + assert_eq!(Conflicts::mark(b"\0binary", b"text", ["a", "b"]), None); + } +} diff --git a/apps/origin-src/src/lib.rs b/apps/origin-src/src/lib.rs index 94161e76..f00f28fc 100644 --- a/apps/origin-src/src/lib.rs +++ b/apps/origin-src/src/lib.rs @@ -1,13 +1,17 @@ //! The origin command line and server. Both share the repository logic in //! this library; `main.rs` only parses arguments and dispatches. +pub mod ancestry; pub mod annotations; pub mod cli; pub mod config; +pub mod conflict; #[cfg(feature = "fixture")] pub mod fixture; pub mod index; pub mod output; +pub mod policy; +pub mod queue; pub mod store; pub mod targets; pub mod tree; diff --git a/apps/origin-src/src/policy.rs b/apps/origin-src/src/policy.rs new file mode 100644 index 00000000..61a67b6f --- /dev/null +++ b/apps/origin-src/src/policy.rs @@ -0,0 +1,621 @@ +//! `policy.toml`, in the snapshot of the `policy` target: the nodes and +//! groups of a repository, and who orders each target and what landing on +//! it requires. The policy in force is the one at the target's head. +//! +//! ```toml +//! [nodes] # each node's ed25519 public key, in hex +//! hosted = "3b6a27bc..." +//! +//! [groups] # users and nodes +//! maintainers = ["user:alice"] +//! runners = ["node:ci-1", "node:ci-2"] +//! +//! [targets.main] +//! authority = "node:hosted" # the one node that orders the target +//! require = [ +//! { kind = "ci.test", verdict = "pass", issuer = "group:runners" }, +//! { kind = "review", verdict = "pass", issuer = "group:maintainers" }, +//! ] +//! +//! [[paths]] +//! path = "secrets" +//! grants = { all = "none", "group:maintainers" = "write" } +//! ``` +//! +//! A target without a table of its own has the authority of `policy`, and +//! requires nothing. Every name a policy refers to must be defined in it, +//! and unknown keys are errors, so a typo never drops a requirement. Path +//! rules are parsed and checked, but not enforced yet. + +use std::collections::BTreeMap; +use std::fmt; +use std::str::FromStr; + +use origin_platform::Platform; +use origin_shared::{ + AcceptanceBody, AcceptanceId, EncodedObject, EntryKind, EntryName, Kind, NodePublicKey, + Principal, PrincipalName, Record, RepoPath, TargetName, Tree, TreeEntry, TreeId, Verdict, +}; +use serde::Deserialize; + +use crate::store::{Store, StoreError}; + +/// The policy of a repository. +#[derive(Debug, Default, Deserialize)] +#[serde(try_from = "PolicyFile")] +pub struct Policy { + nodes: BTreeMap, + groups: BTreeMap>, + targets: BTreeMap, + paths: Vec, +} + +/// Who orders a target, and what landing on it requires. +#[derive(Clone, Debug, PartialEq, Eq)] +struct TargetRules { + authority: PrincipalName, + require: Vec, +} + +/// A statement that landing needs: of `kind`, with `verdict`, from one of +/// `issuer`, about what lands. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Requirement { + /// The statement's kind. + pub kind: Kind, + /// Its verdict. + pub verdict: Verdict, + /// Who may issue it. + pub issuer: Principals, +} + +/// One user or node, or the members of a group: `user:alice`, +/// `node:ci-1` or `group:runners`. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Principals { + /// A user or a node. + One(Principal), + /// The members of a group of the policy. + Group(PrincipalName), +} + +/// A `[[paths]]` rule: what each grantee may do under a path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PathRule { + /// The file or directory the rule covers, with everything under it. + pub path: RepoPath, + /// What each grantee may do there. + pub grants: BTreeMap, +} + +/// Who a path rule grants access to. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Grantee { + /// `all`: everyone. + All, + /// A user, a node or a group's members. + Principals(Principals), +} + +/// What a path rule grants. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Access { + /// Nothing, not even the content. + None, + /// Reading the content. + Read, + /// Proposing revisions that change it. + Write, +} + +impl Policy { + /// The file holding the policy, at the root of the `policy` target. + pub const FILE: &str = "policy.toml"; + + /// The acceptance of `policy` whose `policy.toml` governs the acceptance + /// `body`, `id`: who must have signed it, and what it had to meet. An + /// acceptance of `policy` is governed by the one before it, which it + /// must name, so a policy never authorizes its own landing. Only the + /// first, which has none before it, is governed by its own. Any other + /// acceptance is governed by the policy it names, which the caller + /// checks is on `policy`'s chain. The landing queue checks what it + /// signs with it, and so must whatever verifies a chain. + /// + /// # Errors + /// + /// Fails if the acceptance names another policy than that rule's, or + /// none. + pub fn governing(id: AcceptanceId, body: &AcceptanceBody) -> Result { + if body.target != TargetName::POLICY.id() { + return body.policy.ok_or(PolicyError::Unnamed(body.seq)); + } + match (body.prev, body.policy) { + (None, None) => Ok(id), + (Some(prev), Some(policy)) if policy == prev => Ok(prev), + _ => Err(PolicyError::NotPredecessor(body.seq)), + } + } + + /// The policy in force, at the head of the `policy` target, and the + /// acceptance holding it. + /// + /// # Errors + /// + /// Fails if the repository has no policy, its records cannot be read, + /// or it is invalid. + pub fn in_force(store: &Store

) -> Result<(AcceptanceId, Self), PolicyError> { + let head = store + .head(&TargetName::POLICY)? + .ok_or(PolicyError::Missing)?; + let state = store.get(head.acceptance)?.body().state; + let policy = Self::of_snapshot(store, state)?.ok_or(PolicyError::Missing)?; + Ok((head.acceptance, policy)) + } + + /// The policy in force, or the default if the repository has none. + /// + /// # Errors + /// + /// Fails like [`Policy::in_force`], but for a missing policy. + pub fn load(store: &Store

) -> Result { + match Self::in_force(store) { + Ok((_, policy)) => Ok(policy), + Err(PolicyError::Missing) => Ok(Self::default()), + Err(error) => Err(error), + } + } + + /// The policy in `snapshot`, or `None` if it has no `policy.toml`. + /// + /// # Errors + /// + /// Fails if the snapshot cannot be read, or its `policy.toml` is + /// invalid. + pub fn of_snapshot( + store: &Store

, + snapshot: TreeId, + ) -> Result, PolicyError> { + let Some(&entry) = store.get(snapshot)?.get(Self::FILE) else { + return Ok(None); + }; + let content = store + .try_blob(entry.id)? + .ok_or(StoreError::NotFound(entry.id))?; + let content = String::from_utf8(content).map_err(|_| PolicyError::NotText(snapshot))?; + toml::from_str(&content) + .map(Some) + .map_err(|source| PolicyError::Invalid { + snapshot, + source: Box::new(source), + }) + } + + /// A snapshot holding only `policy` as its `policy.toml`: the file and + /// the tree. + pub(crate) fn snapshot(policy: &str) -> (EncodedObject, EncodedObject) { + let file = EncodedObject::blob(policy.as_bytes()); + let entry = TreeEntry { + kind: EntryKind::File, + id: file.id(), + }; + let name = EntryName::new(Self::FILE).expect("`policy.toml` is a valid name"); + let tree: Tree = [(name, entry)].into_iter().collect(); + (file, tree.encode()) + } + + /// The node that orders `target`: the one its table names, or else the + /// authority of `policy`. `None` if neither has a table. + #[must_use] + pub fn authority(&self, target: &TargetName) -> Option<&PrincipalName> { + self.targets + .get(target) + .or_else(|| self.targets.get(&TargetName::POLICY)) + .map(|rules| &rules.authority) + } + + /// What landing on `target` requires: nothing, if it has no table. + #[must_use] + pub fn requirements(&self, target: &TargetName) -> &[Requirement] { + self.targets + .get(target) + .map_or(&[], |rules| rules.require.as_slice()) + } + + /// The public key of `node`, if the policy lists it. + #[must_use] + pub fn node_key(&self, node: &PrincipalName) -> Option<&NodePublicKey> { + self.nodes.get(node) + } + + /// Whether `principal` is one of `principals`. + #[must_use] + pub fn admits(&self, principals: &Principals, principal: &Principal) -> bool { + match principals { + Principals::One(one) => one == principal, + Principals::Group(group) => self + .groups + .get(group) + .is_some_and(|members| members.contains(principal)), + } + } + + /// The `[[paths]]` rules, in order. + #[must_use] + pub fn paths(&self) -> &[PathRule] { + &self.paths + } +} + +/// `policy.toml` as written, which [`Policy::try_from`] parses and checks. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PolicyFile { + #[serde(default)] + nodes: BTreeMap, + #[serde(default)] + groups: BTreeMap>, + #[serde(default)] + targets: BTreeMap, + #[serde(default)] + paths: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TargetFile { + authority: String, + #[serde(default)] + require: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RequirementFile { + kind: String, + verdict: String, + issuer: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PathRuleFile { + path: String, + grants: BTreeMap, +} + +impl PolicyFile { + /// Parses `value`, which is a `what`. + fn parse(what: &'static str, value: String) -> Result + where + T: FromStr, + T::Err: fmt::Display, + { + value.parse().map_err(|error: T::Err| RuleError::Invalid { + what, + reason: error.to_string(), + value, + }) + } +} + +impl TryFrom for Policy { + type Error = RuleError; + + fn try_from(file: PolicyFile) -> Result { + let nodes = file + .nodes + .into_iter() + .map(|(name, key)| { + Ok(( + PolicyFile::parse("node name", name)?, + PolicyFile::parse("node key", key)?, + )) + }) + .collect::>()?; + let groups = file + .groups + .into_iter() + .map(|(name, members)| { + let members = members + .into_iter() + .map(|member| PolicyFile::parse("group member", member)) + .collect::>()?; + Ok((PolicyFile::parse("group name", name)?, members)) + }) + .collect::>()?; + let mut policy = Self { + nodes, + groups, + targets: BTreeMap::new(), + paths: Vec::new(), + }; + for (name, target) in file.targets { + let name: TargetName = PolicyFile::parse("target name", name)?; + let authority = match PolicyFile::parse("authority", target.authority)? { + Principal::Node(node) if policy.nodes.contains_key(&node) => node, + Principal::Node(node) => return Err(RuleError::UnknownNode { target: name, node }), + user @ Principal::User(_) => { + return Err(RuleError::NotANode { + target: name, + authority: user, + }); + } + }; + let require = target + .require + .into_iter() + .map(|requirement| { + Ok(Requirement { + kind: PolicyFile::parse("kind", requirement.kind)?, + verdict: PolicyFile::parse("verdict", requirement.verdict)?, + issuer: policy.check(PolicyFile::parse("issuer", requirement.issuer)?)?, + }) + }) + .collect::>()?; + policy + .targets + .insert(name, TargetRules { authority, require }); + } + for rule in file.paths { + let grants = rule + .grants + .into_iter() + .map(|(grantee, access)| { + let grantee = match PolicyFile::parse("grantee", grantee)? { + Grantee::Principals(principals) => { + Grantee::Principals(policy.check(principals)?) + } + Grantee::All => Grantee::All, + }; + Ok((grantee, PolicyFile::parse("access", access)?)) + }) + .collect::>()?; + policy.paths.push(PathRule { + path: PolicyFile::parse("path", rule.path)?, + grants, + }); + } + Ok(policy) + } +} + +impl Policy { + /// Checks that the group `principals` names, if any, is defined. + fn check(&self, principals: Principals) -> Result { + match &principals { + Principals::Group(group) if !self.groups.contains_key(group) => { + Err(RuleError::UnknownGroup(group.clone())) + } + _ => Ok(principals), + } + } +} + +impl FromStr for Principals { + type Err = String; + + fn from_str(input: &str) -> Result { + match input.strip_prefix("group:") { + Some(group) => PrincipalName::new(group) + .map(Self::Group) + .map_err(|error| error.to_string()), + None => input.parse().map(Self::One).map_err(|_| { + format!("`{input}` is not `user:`, `node:` or `group:`") + }), + } + } +} + +impl fmt::Display for Principals { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::One(principal) => principal.fmt(f), + Self::Group(group) => write!(f, "group:{group}"), + } + } +} + +impl FromStr for Grantee { + type Err = String; + + fn from_str(input: &str) -> Result { + if input == "all" { + return Ok(Self::All); + } + input + .parse() + .map(Self::Principals) + .map_err(|error| format!("{error}, or `all`")) + } +} + +impl FromStr for Access { + type Err = String; + + fn from_str(input: &str) -> Result { + match input { + "none" => Ok(Self::None), + "read" => Ok(Self::Read), + "write" => Ok(Self::Write), + _ => Err(format!("`{input}` is not `none`, `read` or `write`")), + } + } +} + +impl fmt::Display for Requirement { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} {} from {}", self.kind, self.verdict, self.issuer) + } +} + +/// An invalid rule in `policy.toml`. +#[derive(Debug, PartialEq, Eq, thiserror::Error)] +pub enum RuleError { + /// A value doesn't parse. + #[error("{what}: {reason}")] + Invalid { + /// What the value is. + what: &'static str, + /// The value. + value: String, + /// Why it's invalid. + reason: String, + }, + /// A target's authority isn't a node. + #[error("the authority of `{target}` is {authority}, but only a node can be one")] + NotANode { + /// The target. + target: TargetName, + /// Its authority. + authority: Principal, + }, + /// A target's authority has no key in `[nodes]`. + #[error("the authority of `{target}`, node:{node}, has no key in [nodes]")] + UnknownNode { + /// The target. + target: TargetName, + /// Its authority. + node: PrincipalName, + }, + /// A rule names a group `[groups]` doesn't define. + #[error("group:{0} is not defined in [groups]")] + UnknownGroup(PrincipalName), +} + +/// A failure to read the policy. +#[derive(Debug, thiserror::Error)] +pub enum PolicyError { + /// The repository has no `policy` target, or it holds no `policy.toml`. + #[error("the repository has no policy: `policy` holds no policy.toml")] + Missing, + /// The policy's records cannot be read. + #[error(transparent)] + Store(#[from] StoreError), + /// `policy.toml` isn't UTF-8. + #[error("the policy.toml of snapshot {} is not text", .0.short())] + NotText(TreeId), + /// `policy.toml` isn't a valid policy. + #[error("invalid policy.toml in snapshot {}", snapshot.short())] + Invalid { + /// The snapshot holding it. + snapshot: TreeId, + /// What is wrong, boxed: it's large, and rare. + source: Box, + }, + /// An acceptance of `policy`, at this sequence number, names another + /// policy than the acceptance before it. + #[error("acceptance #{0} of `policy` names another policy than the one before it")] + NotPredecessor(u64), + /// An acceptance of another target, at this sequence number, names no + /// policy. + #[error("acceptance #{0} names no policy")] + Unnamed(u64), +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A policy with every table, whose node `hosted` has the key of the + /// secret `[42; 32]`. + fn full_policy() -> String { + let key = origin_shared::NodeKey::from_bytes(&[42; 32]).public(); + format!( + "[nodes]\nhosted = \"{key}\"\n\ + [groups]\nrunners = [\"node:ci-1\"]\nmaintainers = [\"user:alice\"]\n\ + [targets.policy]\nauthority = \"node:hosted\"\n\ + [targets.main]\nauthority = \"node:hosted\"\nrequire = [\n\ + {{ kind = \"ci.test\", verdict = \"pass\", issuer = \"group:runners\" }},\n\ + {{ kind = \"review\", verdict = \"pass\", issuer = \"user:bob\" }},\n]\n\ + [[paths]]\npath = \"secrets\"\n\ + grants = {{ all = \"none\", \"group:maintainers\" = \"write\" }}\n" + ) + } + + fn principal(principal: &str) -> Principal { + principal.parse().unwrap() + } + + #[test] + fn parses_every_table() { + let policy: Policy = toml::from_str(&full_policy()).unwrap(); + let hosted = PrincipalName::new("hosted").unwrap(); + assert_eq!( + policy.node_key(&hosted), + Some(&origin_shared::NodeKey::from_bytes(&[42; 32]).public()) + ); + let main = TargetName::MAIN; + assert_eq!(policy.authority(&main), Some(&hosted)); + let required: Vec<_> = policy + .requirements(&main) + .iter() + .map(ToString::to_string) + .collect(); + assert_eq!( + required, + [ + "ci.test pass from group:runners", + "review pass from user:bob" + ] + ); + let runners = &policy.requirements(&main)[0].issuer; + assert!(policy.admits(runners, &principal("node:ci-1"))); + assert!( + !policy.admits(runners, &principal("user:ci-1")), + "a user isn't the node of the same name" + ); + + // A target without a table: the policy's authority, no requirements. + let other = TargetName::new("overhaul").unwrap(); + assert_eq!(policy.authority(&other), Some(&hosted)); + assert_eq!(policy.requirements(&other), []); + + let [rule] = policy.paths() else { + panic!("one path rule") + }; + assert_eq!(rule.path, RepoPath::new("secrets").unwrap()); + assert_eq!(rule.grants.get(&Grantee::All), Some(&Access::None)); + } + + #[test] + fn rejects_rules_naming_what_it_does_not_define() { + let invalid = |from: &str, to: &str| { + let toml = full_policy().replace(from, to); + assert_ne!(toml, full_policy(), "{from}"); + toml::from_str::(&toml).unwrap_err().to_string() + }; + for (from, to, expected) in [ + ( + "node:hosted", + "node:nobody", + "node:nobody, has no key in [nodes]", + ), + ( + "authority = \"node:hosted\"\nrequire", + "authority = \"user:hosted\"\nrequire", + "only a node", + ), + ( + "group:runners", + "group:nobody", + "group:nobody is not defined", + ), + ("group:maintainers", "group:x", "group:x is not defined"), + ("issuer", "issuers", "unknown field `issuers`"), + ( + "verdict = \"pass\"", + "verdict = \"ok\"", + "verdict: invalid verdict \"ok\"", + ), + ("\"write\"", "\"admin\"", "access: `admin` is not `none`"), + ( + "node:ci-1", + "ci-1", + "group member: invalid principal \"ci-1\"", + ), + ] { + let error = invalid(from, to); + assert!(error.contains(expected), "{error}"); + } + } +} diff --git a/apps/origin-src/src/queue.rs b/apps/origin-src/src/queue.rs new file mode 100644 index 00000000..8151009d --- /dev/null +++ b/apps/origin-src/src/queue.rs @@ -0,0 +1,1966 @@ +//! The landing queue: how a target's authority lands changes and other +//! targets on it, one acceptance at a time. +//! +//! Landing a change lands its stack: the head of each change below it that +//! isn't on the target yet, from the bottom, with an acceptance each. A +//! change that diverged, that its creator abandoned, or whose head has +//! unresolved conflicts (see [`conflict`](crate::conflict)) is refused. A +//! stack that waits is kept whole, so retrying it lands the rest once the +//! change it waits on can. For each revision: +//! +//! 1. **Rebase.** The changes it made to its base are merged onto the +//! target's tip, file by file ([`Store::merge`]). A revision made on the +//! tip lands as it is; any other is rebased into a new revision of its +//! change, on the tip, by this node. A landing that would leave the +//! target as it is lands nothing: the target already has the change. +//! 2. **Conflicts.** If both sides changed a path, a `core.conflict` +//! statement about the revision lists the paths, and the change waits, +//! blocked, until a new revision of it merges. +//! 3. **Policy.** The policy in force now must name this node as the +//! target's authority. On `policy`, that's the policy of the acceptance +//! before, never the one landing ([`Policy::governing`]). Each of the +//! target's requirements needs a [claim](crate::annotations::Claim) +//! about the revision that lands: results with the same input digest +//! are promoted to it first, and a valid promotion claims what its +//! original does, as its original's issuer. +//! 4. **Missing evidence** leaves the landing waiting, until the statements +//! are issued and the landing is retried. +//! 5. **Otherwise** this node signs an acceptance naming what landed, the +//! evidence and the policy, and appends it to the target's chain. +//! +//! Landing another target works the same way: its tip is merged from the +//! newest acceptances both targets' histories share, and the requirements +//! apply to the merged snapshot. A merge that changes nothing lands +//! nothing if either tip is in the other's history: the target has the +//! source's already, or the source has nothing more. Otherwise it lands, +//! with no evidence then, so the target's history records the source's tip +//! and later merges start from it. +//! +//! The queue of each target lives in `queue//`: a `lock` held +//! while landing, so landings on one target never race, and a file for each +//! landing that waits, `change-` or `target-`, so waiting +//! landings can be listed and retried. + +use std::collections::HashSet; +use std::fmt; +use std::fs; +use std::io; +use std::path::PathBuf; + +use origin_platform::{DurableWrite, LockMode, LockedFile, Native, Platform}; +use origin_shared::{ + AcceptanceBody, AcceptanceId, Base, ChangeId, EncodedObject, Kind, NodeKey, Principal, + PrincipalName, Record, RecordId, RepoPath, Revision, RevisionId, Source, StatementId, TargetId, + TargetName, Tree, TreeId, +}; + +use crate::ancestry::Ancestry; +use crate::annotations::{AnnotateError, Annotations, Claim}; +use crate::conflict::Conflicts; +use crate::policy::{Policy, PolicyError, Requirement}; +use crate::store::{Store, StoreError}; +use crate::targets::Head; + +/// The landing queues of a store's targets, as run by one node. +#[derive(Debug)] +pub struct Queue<'a, P = Native> { + store: &'a Store

, + key: &'a NodeKey, + node: PrincipalName, +} + +/// What a landing lands. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Proposal { + /// A change, as of its head. + Change(ChangeId), + /// Another target, as of its tip. + Target(TargetName), +} + +/// How a landing went. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Landing { + /// It landed. + Landed { + /// What landed. + proposal: Proposal, + /// The revision or acceptance the acceptance names. + source: Source, + /// The acceptance. + acceptance: Head, + }, + /// The target already has it, so nothing landed. + Unchanged(Proposal), + /// It waits, and the queue keeps it. + Pending(Pending), +} + +/// A landing that waits, kept in the queue to be listed and retried. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Pending { + /// The target it lands on. + pub onto: TargetName, + /// What lands. + pub proposal: Proposal, + /// What the evidence must be about: the revision that would land, or + /// the merged snapshot of a target. A conflict is about the revision, + /// or the acceptance, that didn't merge. + pub subject: RecordId, + /// Why it waits. + pub blocked: Blocked, +} + +/// Why a landing waits. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Blocked { + /// The target requires these statements about the subject. + Evidence(Vec), + /// Both sides changed these paths. The `core.conflict` statement lists + /// them too. + Conflict { + /// The statement. + statement: StatementId, + /// The paths, sorted. + paths: Vec, + }, +} + +/// Landings onto one target, holding its queue's lock. +struct Session<'q, 'a, P> { + queue: &'q Queue<'a, P>, + onto: TargetName, + policy: Policy, + /// The acceptance of `policy` holding `policy`. + policy_head: AcceptanceId, + /// The target's latest acceptance, and its snapshot. + tip: Head, + state: TreeId, + annotations: Annotations<'a, P>, + _lock: LockedFile, +} + +/// What a session is about to land. +struct Candidate { + /// What lands. + proposal: Proposal, + /// What the queue keeps if it waits: the landing of the stack it's in. + kept_as: Proposal, + source: Source, + /// What the evidence must be about. + subject: RecordId, + /// The target's snapshot once it lands. + state: TreeId, + /// The trees the merge made, to write first. + trees: Vec, + /// The rebased revision, to write after the trees, if it is one. + revision: Option, +} + +impl<'a, P: Platform> Queue<'a, P> { + /// The queues of `store`, run by the node `node`, which signs with + /// `key`. + pub fn new(store: &'a Store

, key: &'a NodeKey, node: PrincipalName) -> Self { + Self { store, key, node } + } + + /// This node, which issues the statements and revisions the queue + /// makes. + fn principal(&self) -> Principal { + Principal::Node(self.node.clone()) + } + + /// Starts a repository's history: the `policy` target, holding `policy` + /// as its `policy.toml`, and an empty `main`, both of which the policy + /// must have this node order. Returns `main`'s first acceptance. + /// + /// # Errors + /// + /// Fails if the policy is invalid or doesn't make this node the + /// authority of both targets, either target exists, or the records + /// cannot be written. + pub fn init(&self, policy: &str) -> Result { + let (file, tree) = Policy::snapshot(policy); + let empty = Tree::default().encode(); + self.store.write_all(&[file, tree.clone(), empty.clone()])?; + let policy = Self::check_policy(self.store, tree.id().into())?; + for target in [TargetName::POLICY, TargetName::MAIN] { + self.authorize(&policy, &target)?; + } + let genesis = |target: &TargetName, state: &EncodedObject, policy| { + AcceptanceBody { + target: target.id(), + seq: 0, + prev: None, + state: state.id().into(), + landed: Vec::new(), + evidence: Vec::new(), + policy, + } + .sign(self.key) + }; + let policy = self.store.append( + &TargetName::POLICY, + &genesis(&TargetName::POLICY, &tree, None), + )?; + Ok(self.store.append( + &TargetName::MAIN, + &genesis(&TargetName::MAIN, &empty, Some(policy)), + )?) + } + + /// Lands `change`, and the changes below it that `onto` doesn't have + /// yet, from the bottom of the stack. Stops at the first that waits, + /// which the queue keeps as a landing of `change`: retrying it lands + /// the rest of the stack once that one can. + /// + /// # Errors + /// + /// Fails if this node isn't the target's authority, a change in the + /// stack diverged, was abandoned or has unresolved conflicts, or the + /// records cannot be read or written. + pub fn land_change( + &self, + change: ChangeId, + onto: &TargetName, + ) -> Result, LandError> { + let mut session = self.session(onto)?; + let stack = session.stack(change)?; + if stack.is_empty() { + session.forget(&Proposal::Change(change))?; + return Ok(vec![Landing::Unchanged(Proposal::Change(change))]); + } + let proposal = Proposal::Change(change); + let mut landings = Vec::new(); + for (id, revision) in stack { + let landing = session.land_revision(&proposal, id, revision)?; + let waits = matches!(landing, Landing::Pending(_)); + landings.push(landing); + if waits { + break; + } + } + Ok(landings) + } + + /// Lands the tip of `source` on `onto`. + /// + /// # Errors + /// + /// Fails if either target doesn't exist or they're the same, this node + /// isn't the authority of `onto`, or the records cannot be read or + /// written. + pub fn land_target( + &self, + source: &TargetName, + onto: &TargetName, + ) -> Result { + if source == onto { + return Err(LandError::SameTarget(onto.clone())); + } + let mut session = self.session(onto)?; + let tip = self + .store + .head(source)? + .ok_or_else(|| LandError::UnknownTarget(source.clone()))?; + let state = self.store.get(tip.acceptance)?.body().state; + let tips = [session.tip.acceptance, tip.acceptance]; + let history = session.history(&tips)?; + let base = history.merge_base(self.store, &tips)?; + let merge = self.store.merge(base, state, session.state)?; + let proposal = Proposal::Target(source.clone()); + if !merge.conflicts.is_empty() { + let subject = RecordId::Acceptance(tip.acceptance); + return session.block(proposal, subject, &merge.conflicts); + } + // A merge that changes nothing has nothing to record either when + // one tip is in the other's history: the target has the source's + // already, or the source has nothing the target lacks. + let [ours, theirs] = tips; + let either = history.descends(ours, theirs) || history.descends(theirs, ours); + if merge.tree == session.state && either { + session.forget(&proposal)?; + return Ok(Landing::Unchanged(proposal)); + } + // Even a merge that changes nothing lands, so that the tip is in + // the target's history, and later merges start from it. + session.submit(Candidate { + kept_as: proposal.clone(), + proposal, + source: Source::Target(tip.acceptance), + subject: RecordId::Snapshot(merge.tree), + state: merge.tree, + trees: merge.trees, + revision: None, + }) + } + + /// Creates the target `name` at the tip of `from`, with a first + /// acceptance landing it. + /// + /// # Errors + /// + /// Fails if `name` exists, `from` doesn't, this node wouldn't be the new + /// target's authority, or the records cannot be read or written. + pub fn create_target(&self, name: &TargetName, from: &TargetName) -> Result { + let _lock = self.lock(name)?; + if self.store.head(name)?.is_some() { + return Err(LandError::TargetExists(name.clone())); + } + let (policy_head, policy) = Policy::in_force(self.store)?; + self.authorize(&policy, name)?; + let from = self + .store + .head(from)? + .ok_or_else(|| LandError::UnknownTarget(from.clone()))?; + let acceptance = AcceptanceBody { + target: name.id(), + seq: 0, + prev: None, + state: self.store.get(from.acceptance)?.body().state, + landed: vec![Source::Target(from.acceptance)], + evidence: Vec::new(), + policy: Some(policy_head), + } + .sign(self.key); + let acceptance = self.store.append(name, &acceptance)?; + Ok(Head { seq: 0, acceptance }) + } + + /// Every landing that waits, on every target, sorted by target. + /// + /// # Errors + /// + /// Fails if the queues cannot be listed, or a landing cannot be read or + /// is malformed. + pub fn pending(store: &Store

) -> Result, StoreError> { + let root = store.queue_dir(); + let list = |dir: &PathBuf| match fs::read_dir(dir) { + Ok(entries) => entries + .map(|entry| entry.map(|entry| entry.path())) + .collect::>>() + .map_err(|source| StoreError::io(dir, source)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(Vec::new()), + Err(source) => Err(StoreError::io(dir, source)), + }; + let mut pending = Vec::new(); + for dir in list(&root)? { + for path in list(&dir)? { + let name = path.file_name().unwrap_or_default(); + if name == Self::LOCK || DurableWrite::

::is_temporary(name) { + continue; + } + // Read without the lock: a landing that just ended forgot it. + let text = match fs::read_to_string(&path) { + Ok(text) => text, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(source) => return Err(StoreError::io(&path, source)), + }; + let mut landing = + Pending::decode(&text).ok_or_else(|| StoreError::Malformed(path.clone()))?; + if let Blocked::Conflict { statement, paths } = &mut landing.blocked { + *paths = Conflicts::listed(store, &store.get(*statement)?)?.paths(); + } + pending.push(landing); + } + } + pending.sort_by(|a, b| a.onto.cmp(&b.onto)); + Ok(pending) + } + + /// The policy in `state`, about to become the policy in force, if it's + /// a valid one that names an authority for `policy`, so the target + /// never locks up. + fn check_policy(store: &Store

, state: TreeId) -> Result { + let policy = Policy::of_snapshot(store, state) + .and_then(|policy| policy.ok_or(PolicyError::Missing)) + .map_err(LandError::InvalidPolicy)?; + if policy.authority(&TargetName::POLICY).is_none() { + return Err(LandError::NoAuthority(TargetName::POLICY)); + } + Ok(policy) + } + + /// The name of the lock file in each target's queue. + const LOCK: &str = "lock"; + + /// The directory of `target`'s queue, created durably if it's missing. + fn queue_dir(&self, target: TargetId) -> Result { + let root = self.store.queue_dir(); + let dir = root.join(target.to_hex()); + if !dir.is_dir() { + fs::create_dir_all(&dir).map_err(|source| StoreError::io(&dir, source))?; + for parent in [&root, root.parent().unwrap_or(&root)] { + self.store.sync_dir(parent)?; + } + } + Ok(dir) + } + + /// Locks the queue of `target`, waiting for the landing in progress. + fn lock(&self, target: &TargetName) -> Result { + let path = self.queue_dir(target.id())?.join(Self::LOCK); + Ok(LockedFile::open( + &self.store.platform, + &path, + LockMode::Exclusive, + )?) + } + + /// Checks that this node is the authority of `target` under `policy`, + /// with the key it lists. + fn authorize(&self, policy: &Policy, target: &TargetName) -> Result<(), LandError> { + let authority = policy + .authority(target) + .ok_or_else(|| LandError::NoAuthority(target.clone()))?; + if *authority != self.node { + return Err(LandError::NotAuthority { + target: target.clone(), + authority: authority.clone(), + node: self.node.clone(), + }); + } + if policy.node_key(authority) != Some(&self.key.public()) { + return Err(LandError::WrongKey(self.node.clone())); + } + Ok(()) + } + + /// Starts landing on `onto`: locks its queue, then reads the policy in + /// force and the target's tip. + fn session(&self, onto: &TargetName) -> Result, LandError> { + let lock = self.lock(onto)?; + let tip = self + .store + .head(onto)? + .ok_or_else(|| LandError::UnknownTarget(onto.clone()))?; + // A landing on `policy` is governed by the acceptance before it: its + // tip, whatever the policy it lands says. + let (policy_head, policy) = if *onto == TargetName::POLICY { + let state = self.store.get(tip.acceptance)?.body().state; + (tip.acceptance, Self::check_policy(self.store, state)?) + } else { + Policy::in_force(self.store)? + }; + self.authorize(&policy, onto)?; + Ok(Session { + queue: self, + onto: onto.clone(), + policy, + policy_head, + state: self.store.get(tip.acceptance)?.body().state, + tip, + annotations: Annotations::open(self.store)?, + _lock: lock, + }) + } +} + +impl Session<'_, '_, P> { + fn store(&self) -> &Store

{ + self.queue.store + } + + /// The heads to land for `change`, bottom first: the change's head, and + /// below it, following each one's base, the heads of the changes the + /// target doesn't have yet. + fn stack(&self, change: ChangeId) -> Result, LandError> { + let index = self.annotations.index(); + let mut stack = Vec::new(); + let mut seen = HashSet::new(); + let mut next = Some(change); + while let Some(change) = next.take() { + if !seen.insert(change) { + break; + } + let head = match index.heads(change)[..] { + [head] => head, + [] => return Err(LandError::UnknownChange(change)), + ref heads => { + return Err(LandError::Divergent { + change, + heads: heads.to_vec(), + }); + } + }; + let revision: Revision = self.store().get(head)?; + if self.has_landed(head, &revision)? { + break; + } + let claims = self.annotations.claims(RecordId::Revision(head))?; + for claim in &claims { + let statement = claim.statement(); + if statement.kind == Kind::ABANDON && self.store().authorizes(statement)? { + return Err(LandError::Abandoned(change)); + } + } + let conflicts = Conflicts::of(self.store(), &claims, &revision)?; + if !conflicts.is_empty() { + return Err(LandError::Unresolved { + change, + paths: conflicts.paths(), + }); + } + if let Base::Revision(below) = revision.base { + next = Some(self.store().get(below)?.change); + } + stack.push((head, revision)); + } + stack.reverse(); + Ok(stack) + } + + /// Whether `revision`, `id`, landed on the target. A revision only lands + /// on the tip it's based on, so the acceptance after that tip tells. + fn has_landed(&self, id: RevisionId, revision: &Revision) -> Result { + let Base::Acceptance(base) = revision.base else { + return Ok(false); + }; + let base = self.store().get(base)?; + if base.body().target != self.onto.id() { + return Ok(false); + } + let Some(next) = self + .store() + .acceptance_at(&self.onto, base.body().seq + 1)? + else { + return Ok(false); + }; + Ok(self + .store() + .get(next)? + .body() + .landed + .contains(&Source::Revision(id))) + } + + /// Rebases the revision `id` onto the tip and lands it, as part of the + /// landing of `stack`, which is kept if it waits. + fn land_revision( + &mut self, + stack: &Proposal, + id: RevisionId, + revision: Revision, + ) -> Result { + let proposal = Proposal::Change(revision.change); + let base = match revision.base { + Base::Acceptance(id) => self.store().get(id)?.body().state, + Base::Revision(id) => self.store().get(id)?.snapshot, + }; + let merge = self + .store() + .merge(Some(base), revision.snapshot, self.state)?; + if !merge.conflicts.is_empty() { + return self.block(stack.clone(), RecordId::Revision(id), &merge.conflicts); + } + if merge.tree == self.state { + self.forget(&proposal)?; + return Ok(Landing::Unchanged(proposal)); + } + let tip = Base::Acceptance(self.tip.acceptance); + if revision.base == tip { + return self.submit(Candidate { + proposal, + kept_as: stack.clone(), + source: Source::Revision(id), + subject: RecordId::Revision(id), + state: merge.tree, + trees: merge.trees, + revision: None, + }); + } + let rebased = Revision { + predecessors: vec![id], + base: tip, + snapshot: merge.tree, + author: self.queue.principal(), + ..revision + } + .encode(); + let rebased_id = RevisionId::from(rebased.id()); + self.submit(Candidate { + proposal, + kept_as: stack.clone(), + source: Source::Revision(rebased_id), + subject: RecordId::Revision(rebased_id), + state: merge.tree, + trees: merge.trees, + revision: Some(rebased), + }) + } + + /// Checks the policy for `candidate`, and lands it if its evidence is + /// there, or keeps it waiting. Writes the records in layers, each + /// durable before the next refers to it: the trees, then the rebased + /// revision, then the promoted statements and the acceptance. + fn submit(&mut self, candidate: Candidate) -> Result { + let store = self.queue.store; + if !candidate.trees.is_empty() { + store.write_all(&candidate.trees)?; + } + // What a landing on `policy` lands governs the next one, which + // this node must still be the authority of. + let landed_policy = if self.onto == TargetName::POLICY { + self.queue.authorize(&self.policy, &self.onto)?; + Some(Queue::check_policy(store, candidate.state)?) + } else { + None + }; + let mut records: Vec = candidate.revision.into_iter().collect(); + // A landing that changes nothing, which only records another + // target's tip, needs no evidence. + let requirements = if candidate.state == self.state { + Vec::new() + } else { + self.policy.requirements(&self.onto).to_vec() + }; + let mut evidence = Vec::new(); + let mut missing = Vec::new(); + if !requirements.is_empty() { + // Promotion reads the revision. + store.write_all(&records)?; + records.clear(); + let promotions = self + .annotations + .promotions(candidate.subject, &self.queue.principal())?; + let mut claims = self.annotations.claims(candidate.subject)?; + for promotion in promotions { + let encoded = promotion.encode(); + let claim = store.claim(encoded.id().into(), promotion)?; + claims.push(claim.map_err(AnnotateError::from)?); + records.push(encoded); + } + for requirement in requirements { + match self.evidence(&claims, &requirement) { + Some(id) if !evidence.contains(&id) => evidence.push(id), + Some(_) => {} + None => missing.push(requirement), + } + } + } + if !missing.is_empty() { + store.write_all(&records)?; + let pending = Pending { + onto: self.onto.clone(), + proposal: candidate.kept_as, + subject: candidate.subject, + blocked: Blocked::Evidence(missing), + }; + self.keep(&pending)?; + return Ok(Landing::Pending(pending)); + } + + let acceptance = AcceptanceBody { + target: self.onto.id(), + seq: self.tip.seq + 1, + prev: Some(self.tip.acceptance), + state: candidate.state, + landed: vec![candidate.source], + evidence, + policy: Some(self.policy_head), + } + .sign(self.queue.key); + let encoded = acceptance.encode(); + if Policy::governing(encoded.id().into(), acceptance.body())? != self.policy_head { + return Err(PolicyError::NotPredecessor(acceptance.body().seq).into()); + } + records.push(encoded); + store.write_all(&records)?; + let id = store.append(&self.onto, &acceptance)?; + self.tip = Head { + seq: acceptance.body().seq, + acceptance: id, + }; + self.state = candidate.state; + if let Some(policy) = landed_policy { + self.policy = policy; + self.policy_head = id; + } + self.forget(&candidate.proposal)?; + Ok(Landing::Landed { + proposal: candidate.proposal, + source: candidate.source, + acceptance: self.tip, + }) + } + + /// The first of `claims` that meets `requirement`: of the kind and + /// verdict required, from an issuer the policy admits, which for a + /// promotion is its original's. + fn evidence(&self, claims: &[Claim], requirement: &Requirement) -> Option { + claims + .iter() + .find(|claim| { + let claimed = claim.claimed(); + claimed.kind == requirement.kind + && claimed.verdict == Some(requirement.verdict) + && self.policy.admits(&requirement.issuer, &claimed.issuer) + }) + .map(Claim::id) + } + + /// Blocks a landing on `conflicts`: issues a `core.conflict` statement + /// about `subject` listing them, unless this node's latest one says + /// the same, and keeps the landing waiting. + fn block( + &mut self, + proposal: Proposal, + subject: RecordId, + conflicts: &Conflicts, + ) -> Result { + let issuer = self.queue.principal(); + let current = self + .annotations + .current(subject, &Kind::CONFLICT, &issuer)?; + let (payload, conflict) = conflicts.statement(subject, issuer, current); + let unchanged = match current { + Some(id) => self.store().get(id)?.payload == conflict.payload, + None => false, + }; + let statement = match current { + Some(id) if unchanged => id, + _ => self.annotations.issue(&conflict, Some(payload))?, + }; + let pending = Pending { + onto: self.onto.clone(), + proposal, + subject, + blocked: Blocked::Conflict { + statement, + paths: conflicts.paths(), + }, + }; + self.keep(&pending)?; + Ok(Landing::Pending(pending)) + } + + /// The file keeping a waiting landing of `proposal`. + fn pending_path(&self, proposal: &Proposal) -> Result { + let name = match proposal { + Proposal::Change(change) => format!("change-{change}"), + Proposal::Target(target) => format!("target-{}", target.id()), + }; + Ok(self.queue.queue_dir(self.onto.id())?.join(name)) + } + + /// Durably keeps a waiting landing, replacing the last attempt's. + fn keep(&self, pending: &Pending) -> Result<(), StoreError> { + let write = self.store().durable_write(); + write.stage( + self.pending_path(&pending.proposal)?, + pending.encode().as_bytes(), + )?; + Ok(write.commit()?) + } + + /// Durably forgets a landing that waited, if one did. + fn forget(&self, proposal: &Proposal) -> Result<(), StoreError> { + let path = self.pending_path(proposal)?; + match fs::remove_file(&path) { + Ok(()) => self.store().sync_dir(path.parent().unwrap_or(&path)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(StoreError::io(&path, source)), + } + } + + /// The history of `tips`, the target's and another target's, to merge + /// that one from: the state of the newest acceptance both histories + /// share, or the [merge](Ancestry::merge_base) of several, if each + /// lacks another, as after targets landed each other. An acceptance's + /// history is the one before it, and the acceptances of other targets + /// it landed: a target's first lands the one it was created from, so + /// targets created from the same one share it. + /// + /// The nearest shared acceptance can be an older one, through a target + /// that forked early and landed recently, so both histories are read + /// whole, each acceptance once. + fn history(&self, tips: &[AcceptanceId]) -> Result, StoreError> { + Ancestry::walk(tips, |id| { + let acceptance = self.store().get(id)?; + let body = acceptance.body(); + let landed = body.landed.iter().filter_map(|source| match *source { + Source::Target(landed) => Some(landed), + Source::Revision(_) => None, + }); + Ok::<_, StoreError>((landed.chain(body.prev).collect(), body.state)) + }) + } +} + +impl Pending { + /// The first line of every file keeping a waiting landing. + const HEADER: &str = "origin pending 1"; + + /// Encodes the landing as lines: the header, `onto `, `change + /// ` or `target `, `subject `, then a `missing + /// ` line for each requirement it lacks, or + /// `conflict `, whose payload lists the paths. + fn encode(&self) -> String { + let mut lines = vec![Self::HEADER.to_owned(), format!("onto {}", self.onto)]; + lines.push(match &self.proposal { + Proposal::Change(change) => format!("change {change}"), + Proposal::Target(target) => format!("target {target}"), + }); + let (kind, id) = match self.subject { + RecordId::Snapshot(id) => ("snapshot", id.to_hex()), + RecordId::Revision(id) => ("revision", id.to_hex()), + RecordId::Statement(id) => ("statement", id.to_hex()), + RecordId::Acceptance(id) => ("acceptance", id.to_hex()), + }; + lines.push(format!("subject {kind} {id}")); + match &self.blocked { + Blocked::Evidence(missing) => lines.extend(missing.iter().map(|requirement| { + format!( + "missing {} {} {}", + requirement.kind, requirement.verdict, requirement.issuer + ) + })), + Blocked::Conflict { statement, .. } => lines.push(format!("conflict {statement}")), + } + lines.join("\n") + "\n" + } + + /// Decodes what [`Pending::encode`] wrote, without the conflicting + /// paths, or `None` if it's malformed. + fn decode(text: &str) -> Option { + let mut lines = text.lines(); + if lines.next()? != Self::HEADER { + return None; + } + let onto = lines.next()?.strip_prefix("onto ")?.parse().ok()?; + let proposal = match lines.next()?.split_once(' ')? { + ("change", id) => Proposal::Change(id.parse().ok()?), + ("target", name) => Proposal::Target(name.parse().ok()?), + _ => return None, + }; + let subject = match lines.next()?.strip_prefix("subject ")?.split_once(' ')? { + ("snapshot", id) => RecordId::Snapshot(id.parse().ok()?), + ("revision", id) => RecordId::Revision(id.parse().ok()?), + ("statement", id) => RecordId::Statement(id.parse().ok()?), + ("acceptance", id) => RecordId::Acceptance(id.parse().ok()?), + _ => return None, + }; + let mut missing = Vec::new(); + let mut conflict = None; + for line in lines { + match line.split_once(' ')? { + ("missing", requirement) => { + let mut fields = requirement.split(' '); + missing.push(Requirement { + kind: fields.next()?.parse().ok()?, + verdict: fields.next()?.parse().ok()?, + issuer: fields.next()?.parse().ok()?, + }); + } + ("conflict", id) => conflict = Some(id.parse().ok()?), + _ => return None, + } + } + let blocked = match conflict { + Some(statement) => Blocked::Conflict { + statement, + paths: Vec::new(), + }, + None => Blocked::Evidence(missing), + }; + Some(Self { + onto, + proposal, + subject, + blocked, + }) + } +} + +impl fmt::Display for Proposal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Change(change) => write!(f, "change {}", change.short()), + Self::Target(target) => write!(f, "`{target}`"), + } + } +} + +/// A failure to land. +#[derive(Debug, thiserror::Error)] +pub enum LandError { + /// No target has this name. + #[error("no target is named `{0}`")] + UnknownTarget(TargetName), + /// `target new` was given a name already taken. + #[error("a target is already named `{0}`")] + TargetExists(TargetName), + /// A target was landed on itself. + #[error("cannot land `{0}` on itself")] + SameTarget(TargetName), + /// The policy names no authority for the target. + #[error("policy.toml names no authority for `{0}`: give it, or `policy`, a [targets] table")] + NoAuthority(TargetName), + /// Another node orders the target. + #[error( + "node:{authority} is the authority of `{target}`, so only it lands there, and this is \ + node:{node}" + )] + NotAuthority { + /// The target. + target: TargetName, + /// Its authority. + authority: PrincipalName, + /// This node. + node: PrincipalName, + }, + /// The policy lists another key for this node. + #[error("policy.toml lists another key for node:{0} than this node's")] + WrongKey(PrincipalName), + /// No revision of the change is known. + #[error("change {} has no revisions", .0.short())] + UnknownChange(ChangeId), + /// The change has several heads. + #[error( + "change {} is divergent, so it cannot land: its heads are {}. Run `origin converge {}` \ + first", + change.short(), + heads.iter().map(RevisionId::short).collect::>().join(", "), + change.short() + )] + Divergent { + /// The change. + change: ChangeId, + /// Its heads. + heads: Vec, + }, + /// The change was given up. + #[error("change {} was abandoned, so it cannot land", .0.short())] + Abandoned(ChangeId), + /// The change's head has unresolved conflicts. + #[error( + "change {} has unresolved conflicts, so it cannot land: edit {}, or accept them as they \ + are with `origin resolve`", + change.short(), + paths.iter().map(ToString::to_string).collect::>().join(", ") + )] + Unresolved { + /// The change. + change: ChangeId, + /// The unresolved paths. + paths: Vec, + }, + /// Landing on `policy` would leave it without a valid policy. + #[error("landing would leave `policy` without a valid policy.toml")] + InvalidPolicy(#[source] PolicyError), + /// The policy in force cannot be read. + #[error(transparent)] + Policy(#[from] PolicyError), + /// A conflict cannot be recorded. + #[error(transparent)] + Annotate(#[from] AnnotateError), + /// Records cannot be read or written. + #[error(transparent)] + Store(#[from] StoreError), +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + use std::thread; + + use origin_shared::{BlobId, Digest, Inputs, Statement, Verdict}; + + use super::*; + use crate::conflict::Conflicted; + use crate::tree::tests::build; + + /// A repository whose `policy` and `main` the node `queue` orders, with + /// `extra` added to its `policy.toml`. + struct Repo { + _dir: tempfile::TempDir, + store: Store, + key: NodeKey, + } + + impl Repo { + fn new(extra: &str) -> Self { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let key = NodeKey::from_bytes(&[7; 32]); + let repo = Self { + _dir: dir, + store, + key, + }; + repo.queue().init(&repo.policy_toml(extra)).unwrap(); + repo + } + + /// A `policy.toml` naming this node the authority of `policy`, then + /// `extra`. + fn policy_toml(&self, extra: &str) -> String { + format!( + "[nodes]\nqueue = \"{}\"\n[targets.policy]\nauthority = \"node:queue\"\n{extra}", + self.key.public() + ) + } + + /// Writes a snapshot of `files`, contents included. + fn files(&self, files: &[(&str, &str)]) -> TreeId { + let blobs: Vec<_> = files + .iter() + .map(|(_, content)| EncodedObject::blob(content.as_bytes())) + .collect(); + self.store.write_all(&blobs).unwrap(); + build(&self.store, files) + } + + fn queue(&self) -> Queue<'_> { + Queue::new(&self.store, &self.key, PrincipalName::new("queue").unwrap()) + } + + fn tip(&self, target: &TargetName) -> Head { + self.store.head(target).unwrap().unwrap() + } + + fn state(&self, target: &TargetName) -> TreeId { + self.store + .get(self.tip(target).acceptance) + .unwrap() + .body() + .state + } + + /// Writes a revision of `change` with `files`, made on `base`. + fn revision( + &self, + change: u8, + predecessors: &[RevisionId], + base: Base, + files: &[(&str, &str)], + ) -> RevisionId { + let encoded = Revision { + change: ChangeId::from_bytes([change; 16]), + predecessors: predecessors.to_vec(), + base, + deps: Vec::new(), + snapshot: self.files(files), + description: BlobId::from(EncodedObject::blob(b"").id()), + author: "user:ada".parse().unwrap(), + } + .encode(); + self.store + .write_all(std::slice::from_ref(&encoded)) + .unwrap(); + encoded.id().into() + } + + /// A revision of a new change `change` on the tip of `main`, adding + /// `files` to it. + fn change_on_main(&self, change: u8, files: &[(&str, &str)]) -> RevisionId { + let base = Base::Acceptance(self.tip(&TargetName::MAIN).acceptance); + let mut all = self.main_files(); + all.extend( + files + .iter() + .map(|&(path, content)| (path.to_owned(), content.to_owned())), + ); + let all: Vec<_> = all.iter().map(|(p, c)| (p.as_str(), c.as_str())).collect(); + self.revision(change, &[], base, &all) + } + + /// The files of `main`'s tip, with their contents. + fn main_files(&self) -> Vec<(String, String)> { + let empty = build(&self.store, &[]); + self.store + .diff(empty, self.state(&TargetName::MAIN)) + .unwrap() + .into_iter() + .map(|change| { + let content = self + .store + .try_blob(change.new.unwrap().id) + .unwrap() + .unwrap(); + ( + change.path.as_str().to_owned(), + String::from_utf8(content).unwrap(), + ) + }) + .collect() + } + + fn land(&self, change: u8) -> Vec { + self.queue() + .land_change(ChangeId::from_bytes([change; 16]), &TargetName::MAIN) + .unwrap() + } + } + + fn change(id: u8) -> Proposal { + Proposal::Change(ChangeId::from_bytes([id; 16])) + } + + /// The revision a landing landed. + fn landed(landing: &Landing) -> RevisionId { + match landing { + Landing::Landed { + source: Source::Revision(revision), + .. + } => *revision, + other => panic!("expected a landed revision, got {other:?}"), + } + } + + #[test] + fn lands_rebases_and_skips_what_the_target_has() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let a = repo.change_on_main(1, &[("a", "a")]); + let b = repo.change_on_main(2, &[("b", "b")]); + + let [landing] = &repo.land(1)[..] else { + panic!() + }; + assert_eq!(landed(landing), a, "made on the tip, it lands as it is"); + let acceptance = repo + .store + .get(repo.tip(&TargetName::MAIN).acceptance) + .unwrap(); + acceptance.verify(&repo.key.public()).unwrap(); + assert_eq!( + acceptance.body().policy, + Some(repo.tip(&TargetName::POLICY).acceptance) + ); + + let [landing] = &repo.land(2)[..] else { + panic!() + }; + let rebased = landed(landing); + assert_ne!(rebased, b, "made on an older tip, it is rebased"); + let revision = repo.store.get(rebased).unwrap(); + assert_eq!(revision.predecessors, [b]); + assert_eq!(revision.author, "node:queue".parse().unwrap()); + let log = repo.store.log(&TargetName::MAIN).unwrap(); + assert_eq!( + revision.base, + Base::Acceptance(log[1]), + "on the tip it landed after" + ); + assert_eq!( + repo.state(&TargetName::MAIN), + repo.files(&[("a", "a"), ("b", "b")]) + ); + + // Both changes are on `main` now: neither lands again, though `b`'s + // head is the rebased revision and `a`'s the original. + assert_eq!(repo.land(1), [Landing::Unchanged(change(1))]); + assert_eq!(repo.land(2), [Landing::Unchanged(change(2))]); + assert_eq!(repo.tip(&TargetName::MAIN).seq, 2); + } + + #[test] + fn stacks_land_from_the_bottom() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let below = repo.change_on_main(1, &[("a", "a")]); + let above = repo.revision(2, &[], Base::Revision(below), &[("a", "a"), ("b", "b")]); + let landings = repo.land(2); + assert_eq!(landings.len(), 2, "{landings:?}"); + assert_eq!(landed(&landings[0]), below); + assert_ne!(landed(&landings[1]), above, "rebased onto the first"); + assert_eq!( + repo.state(&TargetName::MAIN), + repo.files(&[("a", "a"), ("b", "b")]) + ); + } + + #[test] + fn conflicts_block_until_a_revision_merges() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + repo.change_on_main(1, &[("shared", "one")]); + let two = repo.change_on_main(2, &[("shared", "two")]); + repo.land(1); + + let blocked = repo.land(2); + let [Landing::Pending(pending)] = &blocked[..] else { + panic!("{blocked:?}") + }; + assert_eq!(pending.subject, RecordId::Revision(two)); + let Blocked::Conflict { statement, paths } = &pending.blocked else { + panic!("{pending:?}") + }; + assert_eq!(paths, &[RepoPath::new("shared").unwrap()]); + let conflict = repo.store.get(*statement).unwrap(); + assert_eq!(conflict.kind, Kind::CONFLICT); + assert_eq!(conflict.subject, RecordId::Revision(two)); + assert_eq!(repo.land(2), blocked, "retrying issues the same statement"); + assert_eq!( + Queue::pending(&repo.store).unwrap(), + std::slice::from_ref(pending) + ); + + // A new revision that takes `main`'s version merges. + let tip = Base::Acceptance(repo.tip(&TargetName::MAIN).acceptance); + repo.revision(2, &[two], tip, &[("shared", "one"), ("more", "m")]); + assert!(matches!(repo.land(2)[..], [Landing::Landed { .. }])); + assert_eq!(Queue::pending(&repo.store).unwrap(), []); + } + + /// A `ci.test` result about `subject` from `issuer`, reading `src`. + fn ci_test(subject: RevisionId, issuer: &str) -> Statement { + Statement { + subject: RecordId::Revision(subject), + kind: Kind::new("ci.test").unwrap(), + verdict: Some(Verdict::Pass), + issuer: issuer.parse().unwrap(), + inputs: Some(Inputs { + paths: vec![RepoPath::new("src").unwrap()], + definition: Digest::of(b"cargo test"), + toolchain: Digest::of(b"rustc"), + external: false, + }), + payload: None, + supersedes: None, + promoted_from: None, + } + } + + #[test] + fn missing_evidence_waits_and_promoted_results_count() { + let repo = Repo::new( + "[groups]\nrunners = [\"node:ci\"]\n[targets.main]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"ci.test\", verdict = \"pass\", issuer = \"group:runners\" }]\n", + ); + let code = repo.change_on_main(1, &[("src/lib.rs", "fn a() {}")]); + let docs = repo.change_on_main(2, &[("docs/guide.md", "# Guide")]); + let waiting = repo.land(1); + let [Landing::Pending(pending)] = &waiting[..] else { + panic!("{waiting:?}") + }; + assert!(matches!(&pending.blocked, Blocked::Evidence(missing) if missing.len() == 1)); + assert_eq!( + Queue::pending(&repo.store).unwrap(), + std::slice::from_ref(pending) + ); + + // A user outside the group doesn't count; the runner does. + let annotations = Annotations::open(&repo.store).unwrap(); + annotations + .issue(&ci_test(code, "user:mallory"), None) + .unwrap(); + assert_eq!(repo.land(1), waiting); + let result = annotations.issue(&ci_test(code, "node:ci"), None).unwrap(); + let [landing] = &repo.land(1)[..] else { + panic!() + }; + let acceptance = repo + .store + .get(repo.tip(&TargetName::MAIN).acceptance) + .unwrap(); + assert_eq!(landed(landing), code); + assert_eq!(acceptance.body().evidence, [result]); + assert_eq!(Queue::pending(&repo.store).unwrap(), []); + + // The docs change is rebased onto `src` as tested, so the result is + // promoted to it, and counts as the runner's. + let [landing] = &repo.land(2)[..] else { + panic!() + }; + let rebased = landed(landing); + assert_ne!(rebased, docs); + let acceptance = repo + .store + .get(repo.tip(&TargetName::MAIN).acceptance) + .unwrap(); + let [evidence] = acceptance.body().evidence[..] else { + panic!() + }; + let promoted = repo.store.get(evidence).unwrap(); + assert_eq!(promoted.subject, RecordId::Revision(rebased)); + assert_eq!(promoted.promoted_from, Some(result)); + assert_eq!(promoted.issuer, "node:queue".parse().unwrap()); + } + + #[test] + fn refuses_divergent_and_abandoned_changes() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let first = repo.change_on_main(1, &[("a", "a")]); + let base = Base::Acceptance(repo.tip(&TargetName::MAIN).acceptance); + repo.revision(1, &[first], base, &[("a", "left")]); + repo.revision(1, &[first], base, &[("a", "right")]); + let main = TargetName::MAIN; + let error = repo + .queue() + .land_change(ChangeId::from_bytes([1; 16]), &main) + .unwrap_err(); + assert!(matches!(error, LandError::Divergent { .. }), "{error}"); + + let given_up = repo.change_on_main(2, &[("b", "b")]); + let abandon = Statement { + subject: RecordId::Revision(given_up), + kind: Kind::ABANDON, + verdict: None, + issuer: "user:ada".parse().unwrap(), + inputs: None, + payload: None, + supersedes: None, + promoted_from: None, + }; + repo.store.write_all(&[abandon.encode()]).unwrap(); + let error = repo + .queue() + .land_change(ChangeId::from_bytes([2; 16]), &main) + .unwrap_err(); + assert!(matches!(error, LandError::Abandoned(_)), "{error}"); + } + + #[test] + fn only_the_authority_lands() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + repo.change_on_main(1, &[("a", "a")]); + let change = ChangeId::from_bytes([1; 16]); + let other = Queue::new(&repo.store, &repo.key, PrincipalName::new("other").unwrap()); + let error = other.land_change(change, &TargetName::MAIN).unwrap_err(); + assert!(matches!(error, LandError::NotAuthority { .. }), "{error}"); + + let impostor = NodeKey::from_bytes(&[8; 32]); + let queue = Queue::new(&repo.store, &impostor, PrincipalName::new("queue").unwrap()); + let error = queue.land_change(change, &TargetName::MAIN).unwrap_err(); + assert!(matches!(error, LandError::WrongKey(_)), "{error}"); + assert_eq!(repo.tip(&TargetName::MAIN).seq, 0); + } + + #[test] + fn concurrent_landings_serialize() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let (proposers, each) = (6_u8, 3_u8); + let changes: Vec = (0..proposers * each).map(|change| change + 1).collect(); + for &change in &changes { + repo.change_on_main(change, &[(&format!("file-{change}"), "x")]); + } + let landed = Mutex::new(0); + thread::scope(|scope| { + for chunk in changes.chunks(usize::from(each)) { + let (repo, landed) = (&repo, &landed); + scope.spawn(move || { + for &change in chunk { + assert!(matches!(repo.land(change)[..], [Landing::Landed { .. }])); + *landed.lock().unwrap() += 1; + } + }); + } + }); + assert_eq!(*landed.lock().unwrap(), changes.len()); + let log = repo.store.log(&TargetName::MAIN).unwrap(); + assert_eq!(log.len(), changes.len() + 1); + for pair in log.windows(2) { + assert_eq!(repo.store.get(pair[1]).unwrap().body().prev, Some(pair[0])); + } + assert_eq!(repo.main_files().len(), changes.len(), "every file landed"); + } + + #[test] + fn targets_land_on_each_other() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let (main, overhaul) = (TargetName::MAIN, TargetName::new("overhaul").unwrap()); + repo.change_on_main(1, &[("shared", "v1")]); + repo.land(1); + let queue = repo.queue(); + queue.create_target(&overhaul, &main).unwrap(); + assert!(matches!( + queue.create_target(&overhaul, &main), + Err(LandError::TargetExists(_)) + )); + assert_eq!(repo.state(&overhaul), repo.state(&main)); + + // Work lands on the overhaul while `main` moves on elsewhere. + let on_overhaul = Base::Acceptance(repo.tip(&overhaul).acceptance); + repo.revision(2, &[], on_overhaul, &[("shared", "v2"), ("new", "n")]); + queue + .land_change(ChangeId::from_bytes([2; 16]), &overhaul) + .unwrap(); + repo.change_on_main(3, &[("other", "o")]); + repo.land(3); + + // Catching up merges `main`'s change since the fork; finishing + // lands the overhaul on `main`. + assert!(matches!( + queue.land_target(&main, &overhaul).unwrap(), + Landing::Landed { .. } + )); + let landing = queue.land_target(&overhaul, &main).unwrap(); + assert!(matches!( + landing, + Landing::Landed { + source: Source::Target(_), + .. + } + )); + let expected = repo.files(&[("shared", "v2"), ("new", "n"), ("other", "o")]); + assert_eq!(repo.state(&main), expected); + assert_eq!(repo.state(&overhaul), expected); + assert_eq!( + queue.land_target(&overhaul, &main).unwrap(), + Landing::Unchanged(Proposal::Target(overhaul.clone())) + ); + assert!(matches!( + queue.land_target(&main, &main), + Err(LandError::SameTarget(_)) + )); + } + + #[test] + fn policy_landings_keep_a_valid_policy() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let policy = TargetName::POLICY; + let tip = Base::Acceptance(repo.tip(&policy).acceptance); + for (change, toml) in [(1, "not = [valid"), (2, "[nodes]\n")] { + repo.revision(change, &[], tip, &[("policy.toml", toml)]); + let result = repo + .queue() + .land_change(ChangeId::from_bytes([change; 16]), &policy); + assert!( + matches!( + result, + Err(LandError::InvalidPolicy(_) | LandError::NoAuthority(_)) + ), + "{result:?}" + ); + } + // A valid edit lands, and later landings are under it. + let requiring = repo.policy_toml( + "[targets.main]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"review\", verdict = \"pass\", issuer = \"user:bob\" }]\n", + ); + let encoded = Revision { + change: ChangeId::from_bytes([3; 16]), + predecessors: Vec::new(), + base: tip, + deps: Vec::new(), + snapshot: repo.files(&[("policy.toml", &requiring)]), + description: BlobId::from(EncodedObject::blob(b"").id()), + author: "user:ada".parse().unwrap(), + } + .encode(); + repo.store + .write_all(std::slice::from_ref(&encoded)) + .unwrap(); + let landings = repo + .queue() + .land_change(ChangeId::from_bytes([3; 16]), &policy) + .unwrap(); + assert!( + matches!(landings[..], [Landing::Landed { .. }]), + "{landings:?}" + ); + repo.change_on_main(4, &[("a", "a")]); + assert!(matches!(repo.land(4)[..], [Landing::Pending(_)])); + } + + #[test] + fn pending_landings_round_trip() { + let pending = Pending { + onto: TargetName::new("release/v3").unwrap(), + proposal: Proposal::Target(TargetName::MAIN), + subject: RecordId::Snapshot(Tree::default().encode().id().into()), + blocked: Blocked::Evidence(vec![Requirement { + kind: Kind::new("ci.test").unwrap(), + verdict: Verdict::Pass, + issuer: "group:runners".parse().unwrap(), + }]), + }; + assert_eq!(Pending::decode(&pending.encode()), Some(pending.clone())); + let conflict = Pending { + proposal: change(9), + blocked: Blocked::Conflict { + statement: EncodedObject::blob(b"conflict").id().into(), + paths: Vec::new(), + }, + ..pending + }; + assert_eq!(Pending::decode(&conflict.encode()), Some(conflict)); + assert_eq!(Pending::decode("origin pending 2\n"), None); + } + + /// A repository whose `main` requires a passing `ci.test` from + /// `user:ci`. + fn requiring_ci() -> Repo { + Repo::new( + "[targets.main]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"ci.test\", verdict = \"pass\", issuer = \"user:ci\" }]\n", + ) + } + + #[test] + fn forged_promotions_are_no_evidence() { + let repo = requiring_ci(); + let failed = repo.change_on_main(1, &[("src/lib.rs", "bad")]); + let mut original = ci_test(failed, "user:ci"); + original.verdict = Some(Verdict::Fail); + let id = Annotations::open(&repo.store) + .unwrap() + .issue(&original, None) + .unwrap(); + // Eve wraps the trusted failure as a pass, written as a fetch would. + let forged = Statement { + verdict: Some(Verdict::Pass), + issuer: "user:eve".parse().unwrap(), + promoted_from: Some(id), + ..original.clone() + }; + // And carries a result that read the network to another change. + let old = repo.change_on_main(3, &[("src/lib.rs", "bad")]); + let docs = repo.change_on_main(2, &[("src/lib.rs", "bad"), ("docs", "d")]); + let mut external = ci_test(old, "user:ci"); + external.inputs.as_mut().unwrap().external = true; + let external_id: StatementId = external.encode().id().into(); + let reused = Statement { + subject: RecordId::Revision(docs), + issuer: "user:eve".parse().unwrap(), + promoted_from: Some(external_id), + ..external.clone() + }; + repo.store.write_all(&[external.encode()]).unwrap(); + repo.store + .write_all(&[forged.encode(), reused.encode()]) + .unwrap(); + assert!(matches!(repo.land(1)[..], [Landing::Pending(_)])); + assert!(matches!(repo.land(2)[..], [Landing::Pending(_)])); + } + + #[test] + fn replaced_results_are_promoted_again() { + let repo = requiring_ci(); + let old = repo.change_on_main(1, &[("src/lib.rs", "code")]); + let new = repo.change_on_main(2, &[("src/lib.rs", "code"), ("docs", "new")]); + let original = ci_test(old, "user:ci"); + let annotations = Annotations::open(&repo.store).unwrap(); + let id = annotations.issue(&original, None).unwrap(); + Annotations::open(&repo.store) + .unwrap() + .promote(RecordId::Revision(new), &"node:queue".parse().unwrap()) + .unwrap(); + // The same passing result runs again, replacing the one promoted. + let rerun = Statement { + supersedes: Some(id), + ..original + }; + annotations.issue(&rerun, None).unwrap(); + let landings = repo.land(2); + assert!( + matches!(landings[..], [Landing::Landed { .. }]), + "{landings:?}" + ); + } + + #[test] + fn only_the_creator_of_a_change_abandons_it() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let revision = repo.change_on_main(1, &[("file", "work")]); + let abandon = |issuer: &str| Statement { + subject: RecordId::Revision(revision), + kind: Kind::ABANDON, + verdict: None, + issuer: issuer.parse().unwrap(), + inputs: None, + payload: None, + supersedes: None, + promoted_from: None, + }; + repo.store + .write_all(&[abandon("user:eve").encode()]) + .unwrap(); + assert!(matches!(repo.land(1)[..], [Landing::Landed { .. }])); + } + + #[test] + fn unresolved_conflicts_never_land() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let marked = "<<<<<<< a\n=======\n>>>>>>> b\n"; + let revision = repo.change_on_main(1, &[("file", marked)]); + let path = RepoPath::new("file").unwrap(); + let conflicts = |content: &str| { + Conflicts::new(vec![Conflicted { + path: path.clone(), + written: Some(EncodedObject::blob(content.as_bytes()).id()), + }]) + }; + // From someone else, it says nothing about the revision's files. + let foreign = conflicts(marked).statement( + RecordId::Revision(revision), + "user:eve".parse().unwrap(), + None, + ); + // From its author, it does. + let (payload, statement) = conflicts(marked).statement( + RecordId::Revision(revision), + "user:ada".parse().unwrap(), + None, + ); + repo.store + .write_all(&[foreign.0, foreign.1.encode()]) + .unwrap(); + let mut annotations = Annotations::open(&repo.store).unwrap(); + assert_eq!( + annotations + .claims(RecordId::Revision(revision)) + .unwrap() + .len(), + 1 + ); + annotations.issue(&statement, Some(payload)).unwrap(); + annotations = Annotations::open(&repo.store).unwrap(); + assert_eq!( + annotations + .claims(RecordId::Revision(revision)) + .unwrap() + .len(), + 2 + ); + let error = repo + .queue() + .land_change(ChangeId::from_bytes([1; 16]), &TargetName::MAIN) + .unwrap_err(); + assert!( + matches!(&error, LandError::Unresolved { paths, .. } if *paths == [path.clone()]), + "{error}" + ); + + // A revision whose file is no longer what the conflict wrote there + // is resolved, whatever the file holds. + let edited = repo.change_on_main(2, &[("file", "Heading\n=======\n")]); + let (payload, statement) = conflicts(marked).statement( + RecordId::Revision(edited), + "user:ada".parse().unwrap(), + None, + ); + annotations.issue(&statement, Some(payload)).unwrap(); + assert!(matches!(repo.land(2)[..], [Landing::Landed { .. }])); + } + + #[test] + fn sibling_targets_merge_from_where_they_forked() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + repo.change_on_main(1, &[("keep", "k"), ("delete", "d")]); + repo.land(1); + let (a, b) = (TargetName::new("a").unwrap(), TargetName::new("b").unwrap()); + let queue = repo.queue(); + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + // `a` deletes a file, `b` adds one: neither ever landed the other. + let on = |target| Base::Acceptance(repo.tip(target).acceptance); + repo.revision(2, &[], on(&a), &[("keep", "k")]); + queue + .land_change(ChangeId::from_bytes([2; 16]), &a) + .unwrap(); + repo.revision( + 3, + &[], + on(&b), + &[("keep", "k"), ("delete", "d"), ("other", "o")], + ); + queue + .land_change(ChangeId::from_bytes([3; 16]), &b) + .unwrap(); + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!(repo.state(&b), repo.files(&[("keep", "k"), ("other", "o")])); + } + + #[test] + fn a_merge_that_changes_nothing_still_records_the_tip() { + // `b` requires tests, which a landing changing nothing doesn't. + let repo = Repo::new( + "[targets.b]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"ci.test\", verdict = \"pass\", issuer = \"user:ci\" }]\n", + ); + let queue = repo.queue(); + let annotations = Annotations::open(&repo.store).unwrap(); + // Results without inputs, which are never promoted to a merge. + let pass = |subject| { + let statement = Statement { + subject, + inputs: None, + ..ci_test(RevisionId::from(EncodedObject::blob(b"").id()), "user:ci") + }; + annotations.issue(&statement, None).unwrap(); + }; + let (a, b) = (TargetName::new("a").unwrap(), TargetName::new("b").unwrap()); + let land = |change: u8, target: &TargetName, files: &[(&str, &str)]| { + let on = Base::Acceptance(repo.tip(target).acceptance); + pass(RecordId::Revision(repo.revision(change, &[], on, files))); + queue + .land_change(ChangeId::from_bytes([change; 16]), target) + .unwrap(); + }; + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + // Both add the same file: `a`'s tip lands on `b`, changing nothing. + land(1, &a, &[("f", "f")]); + land(2, &b, &[("f", "f")]); + let state = repo.state(&b); + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!(repo.state(&b), state); + assert_eq!( + queue.land_target(&a, &b).unwrap(), + Landing::Unchanged(Proposal::Target(a.clone())) + ); + + // So once `a` deletes the file, landing it on `b` deletes it there. + land(3, &a, &[]); + let landing = queue.land_target(&a, &b).unwrap(); + let Landing::Pending(pending) = landing else { + panic!("a deletion waits for its evidence: {landing:?}") + }; + pass(pending.subject); + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!(repo.state(&b), repo.files(&[])); + + // A target and the one it was made from, neither changing: landing + // either on the other records nothing, however often. + let c = TargetName::new("c").unwrap(); + queue.create_target(&c, &TargetName::MAIN).unwrap(); + for _ in 0..2 { + for (from, onto) in [(&c, &TargetName::MAIN), (&TargetName::MAIN, &c)] { + assert_eq!( + queue.land_target(from, onto).unwrap(), + Landing::Unchanged(Proposal::Target(from.clone())) + ); + } + } + } + + #[test] + fn targets_merge_from_their_newest_shared_acceptance() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let queue = repo.queue(); + let [a, b, c, d] = ["a", "b", "c", "d"].map(|name| TargetName::new(name).unwrap()); + let land = |change: u8, target: &TargetName, files: &[(&str, &str)]| { + let on = Base::Acceptance(repo.tip(target).acceptance); + repo.revision(change, &[], on, files); + queue + .land_change(ChangeId::from_bytes([change; 16]), target) + .unwrap(); + }; + // `c` and `d` fork from an empty `main`, `a` and `b` from the next + // `main`, which adds a file. + queue.create_target(&c, &TargetName::MAIN).unwrap(); + queue.create_target(&d, &TargetName::MAIN).unwrap(); + repo.change_on_main(1, &[("delete", "d")]); + repo.land(1); + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + // `a` deletes it, `b` keeps it, and both move on. + for n in 0..4 { + let counter = n.to_string(); + land(10 + n, &a, &[("a", &counter)]); + land(20 + n, &b, &[("delete", "d"), ("b", &counter)]); + } + // Then each lands a side line: the empty `main` is nearer. + land(30, &c, &[("c", "c")]); + land(31, &d, &[("d", "d")]); + assert!(matches!( + queue.land_target(&c, &a).unwrap(), + Landing::Landed { .. } + )); + assert!(matches!( + queue.land_target(&d, &b).unwrap(), + Landing::Landed { .. } + )); + + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!( + repo.state(&b), + repo.files(&[("a", "3"), ("b", "3"), ("c", "c"), ("d", "d")]), + "the deletion stands" + ); + } + + #[test] + fn criss_crossed_targets_merge_what_their_bases_agree_on() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let queue = repo.queue(); + let [a, b] = ["a", "b"].map(|name| TargetName::new(name).unwrap()); + let land = |change: u8, target: &TargetName, files: &[(&str, &str)]| { + let on = Base::Acceptance(repo.tip(target).acceptance); + repo.revision(change, &[], on, files); + queue + .land_change(ChangeId::from_bytes([change; 16]), target) + .unwrap(); + }; + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + land(1, &a, &[("a", "0"), ("shared", "a")]); + land(2, &b, &[("b", "0"), ("shared", "b")]); + // Each lands the other's tip at once, resolving `shared` its own + // way: two merge bases, which disagree on it. + let tips = [repo.tip(&a), repo.tip(&b)]; + let policy = Some(repo.tip(&TargetName::POLICY).acceptance); + for (target, prev, other, shared) in + [(&a, tips[0], tips[1], "a"), (&b, tips[1], tips[0], "b")] + { + let merged = AcceptanceBody { + target: target.id(), + seq: prev.seq + 1, + prev: Some(prev.acceptance), + state: repo.files(&[("a", "0"), ("b", "0"), ("shared", shared)]), + landed: vec![Source::Target(other.acceptance)], + evidence: vec![], + policy, + } + .sign(&repo.key); + repo.store.append(target, &merged).unwrap(); + } + // Then each changes another file. + land(3, &a, &[("a", "1"), ("b", "0"), ("shared", "a")]); + land(4, &b, &[("a", "0"), ("b", "1"), ("shared", "b")]); + + let Landing::Pending(pending) = queue.land_target(&a, &b).unwrap() else { + panic!("the bases disagree on `shared`, so it can't be merged blindly"); + }; + assert!( + matches!(&pending.blocked, Blocked::Conflict { paths, .. } if paths.iter().map(RepoPath::as_str).eq(["shared"])), + "{pending:?}" + ); + // Once `a` takes `b`'s side, the disjoint edits merge. + land(5, &a, &[("a", "1"), ("b", "0"), ("shared", "b")]); + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!( + repo.state(&b), + repo.files(&[("a", "1"), ("b", "1"), ("shared", "b")]) + ); + } + + #[test] + fn bases_that_disagree_on_a_kind_conflict_on_any_change_under_it() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let queue = repo.queue(); + let land = |change: u8, target: &TargetName, files: &[(&str, &str)]| { + let on = Base::Acceptance(repo.tip(target).acceptance); + repo.revision(change, &[], on, files); + queue + .land_change(ChangeId::from_bytes([change; 16]), target) + .unwrap(); + }; + let [a, b] = ["a", "b"].map(|name| TargetName::new(name).unwrap()); + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + land(1, &a, &[("p", "file")]); + land(2, &b, &[("p/delete", "original")]); + // Each lands the other's tip at once, both choosing the directory: + // two merge bases, one with a file at `p`, the other a directory. + let tips = [repo.tip(&a), repo.tip(&b)]; + let policy = Some(repo.tip(&TargetName::POLICY).acceptance); + for (target, prev, other) in [(&a, tips[0], tips[1]), (&b, tips[1], tips[0])] { + let merged = AcceptanceBody { + target: target.id(), + seq: prev.seq + 1, + prev: Some(prev.acceptance), + state: repo.files(&[("p/delete", "original")]), + landed: vec![Source::Target(other.acceptance)], + evidence: vec![], + policy, + } + .sign(&repo.key); + repo.store.append(target, &merged).unwrap(); + } + // `a` deletes `p/delete`, `b` keeps it and adds beside it. + land(3, &a, &[("a", "new")]); + land(4, &b, &[("p/delete", "original"), ("p/new", "new")]); + + // What `p` was is unknown, so the deletion can't be told from an + // addition: it conflicts, rather than bring `p/delete` back. + let Landing::Pending(pending) = queue.land_target(&a, &b).unwrap() else { + panic!("the bases disagree on `p`, so it can't be merged blindly"); + }; + assert!( + matches!(&pending.blocked, Blocked::Conflict { paths, .. } if paths.iter().map(RepoPath::as_str).eq(["p"])), + "{pending:?}" + ); + } + + #[test] + fn a_waiting_stack_lands_whole_once_it_can() { + let repo = requiring_ci(); + let lower = repo.change_on_main(1, &[("a", "a")]); + repo.revision(2, &[], Base::Revision(lower), &[("a", "a"), ("b", "b")]); + let [Landing::Pending(pending)] = &repo.land(2)[..] else { + panic!("the lower change waits") + }; + assert_eq!(pending.subject, RecordId::Revision(lower)); + assert_eq!( + Queue::pending(&repo.store).unwrap(), + [Pending { + proposal: change(2), + ..pending.clone() + }], + "kept as the landing of the whole stack" + ); + + // Retrying it, as once the evidence arrives, lands both: the result + // is promoted to the upper change, which doesn't change `src`. + let annotations = Annotations::open(&repo.store).unwrap(); + annotations.issue(&ci_test(lower, "user:ci"), None).unwrap(); + let landings = repo.land(2); + assert_eq!(landed(&landings[0]), lower); + assert!( + matches!(landings[1], Landing::Landed { proposal: ref p, .. } if *p == change(2)), + "{landings:?}" + ); + assert_eq!(Queue::pending(&repo.store).unwrap(), []); + } + + #[test] + fn each_policy_landing_is_governed_by_the_one_before() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let policy = TargetName::POLICY; + // The first edit requires a review to land on `policy`; the second, + // stacked on it, must then wait for one. + let requiring = repo + .policy_toml("[targets.main]\nauthority = \"node:queue\"\n") + .replace( + "[targets.policy]\nauthority = \"node:queue\"\n", + "[targets.policy]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"review\", verdict = \"pass\", issuer = \"user:bob\" }]\n", + ); + let first = repo.revision( + 1, + &[], + Base::Acceptance(repo.tip(&policy).acceptance), + &[("policy.toml", &requiring)], + ); + let open = repo.policy_toml("[targets.main]\nauthority = \"node:queue\"\n"); + repo.revision(2, &[], Base::Revision(first), &[("policy.toml", &open)]); + let landings = repo + .queue() + .land_change(ChangeId::from_bytes([2; 16]), &policy) + .unwrap(); + assert_eq!(landed(&landings[0]), first); + assert!(matches!(landings[1], Landing::Pending(_)), "{landings:?}"); + + let log = repo.store.log(&policy).unwrap(); + for (seq, &id) in log.iter().enumerate() { + let body = repo.store.get(id).unwrap().body().clone(); + let governing = Policy::governing(id, &body).unwrap(); + assert_eq!(governing, if seq == 0 { id } else { log[seq - 1] }); + } + // An acceptance naming any other policy than the one before it is + // refused, its own included. + let mut body = repo.store.get(log[1]).unwrap().body().clone(); + body.policy = Some(log[1]); + assert!(matches!( + Policy::governing(log[1], &body), + Err(PolicyError::NotPredecessor(1)) + )); + } +} diff --git a/apps/origin-src/src/store/mod.rs b/apps/origin-src/src/store/mod.rs index ed782a13..81861f1f 100644 --- a/apps/origin-src/src/store/mod.rs +++ b/apps/origin-src/src/store/mod.rs @@ -1,6 +1,7 @@ //! A repository's storage on disk, under its `.origin` directory: -//! content-addressed objects here, and target chains in -//! [`targets`](crate::targets). +//! content-addressed objects here, target chains in +//! [`targets`](crate::targets), and pending landings in +//! [`queue`](crate::queue). //! //! Each object lives in its own zstd-compressed file at //! `objects/loose//`. Objects are @@ -185,6 +186,10 @@ impl Store

{ self.root.join("index") } + pub(crate) fn queue_dir(&self) -> PathBuf { + self.root.join("queue") + } + pub(crate) fn path(&self, id: ObjectId) -> PathBuf { let hex = id.to_hex(); let mut path = self.loose(); diff --git a/apps/origin-src/src/targets.rs b/apps/origin-src/src/targets.rs index 05209177..a41eba14 100644 --- a/apps/origin-src/src/targets.rs +++ b/apps/origin-src/src/targets.rs @@ -20,6 +20,7 @@ use std::fmt; use std::fs::{self, File}; use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use std::slice; @@ -177,6 +178,32 @@ impl Store

{ Ok(log) } + /// The acceptance at position `seq` of `target`'s log, or `None` if + /// none is published there yet. Reads one line of the log. + /// + /// # Errors + /// + /// Fails if the target's files cannot be read or are malformed. + pub fn acceptance_at( + &self, + target: &TargetName, + seq: u64, + ) -> Result, StoreError> { + let files = self.target_files(target.id()); + let Some(head) = files.read_head()? else { + return Ok(None); + }; + if seq >= head.seq { + return Ok((seq == head.seq).then_some(head.acceptance)); + } + let path = files.log(); + let mut line = [0; TargetFiles::LINE]; + File::open(&path) + .and_then(|log| log.read_exact_at(&mut line, seq * TargetFiles::LINE as u64)) + .map_err(|source| StoreError::io(&path, source))?; + Ok(files.parse_log(&line)?.first().copied()) + } + /// The names of every target, in no particular order. /// /// # Errors @@ -363,6 +390,10 @@ mod tests { }) ); assert_eq!(store.log(&main).unwrap(), ids); + for (seq, &id) in (0..).zip(&ids) { + assert_eq!(store.acceptance_at(&main, seq).unwrap(), Some(id)); + } + assert_eq!(store.acceptance_at(&main, 3).unwrap(), None); let last = store.get(ids[2]).unwrap(); assert_eq!(last.body().prev, Some(ids[1])); last.verify(&key().public()).unwrap(); diff --git a/apps/origin-src/src/tree.rs b/apps/origin-src/src/tree.rs index 1e37cf63..cc59b90d 100644 --- a/apps/origin-src/src/tree.rs +++ b/apps/origin-src/src/tree.rs @@ -1,11 +1,15 @@ -//! Comparing trees, and looking up what they hold. +//! Comparing trees, merging them, and looking up what they hold. use std::collections::HashMap; use std::collections::hash_map::Entry; use origin_platform::Platform; -use origin_shared::{Digest, EntryKind, EntryName, Inputs, RepoPath, Tree, TreeEntry, TreeId}; +use origin_shared::{ + Digest, EncodedObject, EntryKind, EntryName, Inputs, ObjectId, Record, RepoPath, Tree, + TreeEntry, TreeId, +}; +use crate::conflict::{Conflicted, Conflicts}; use crate::store::{Store, StoreError}; /// A file that differs between two trees. @@ -82,6 +86,255 @@ impl Store

{ } } +/// Two trees merged with the changes each made to a common one. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Merge { + /// The merged tree: both sides' changes, and `ours` where they conflict. + pub tree: TreeId, + /// The trees the merge made, which the store may not have yet. + pub trees: Vec, + /// The files with conflict markers a [marked](Store::merge_marked) + /// merge made, which the store may not have yet either. + pub files: Vec, + /// The paths both sides changed, each its own way, with what the merge + /// left there. + pub conflicts: Conflicts, +} + +/// What a merge leaves where both sides changed a path, each its own way. +#[derive(Clone, Copy, Debug)] +enum OnConflict<'l> { + /// `ours`. + KeepOurs, + /// Both versions between conflict markers naming the sides with these + /// labels, see [`Conflicts::mark`]. + Mark([&'l str; 2]), + /// [`Store::UNKNOWN`], a version neither side can have. + Unknown, +} + +impl Store

{ + /// Merges `ours` and `theirs`, two trees made from `base`, file by file: + /// a path one side changed takes that side's version, and a path both + /// changed, each its own way, is a conflict, where the merge keeps + /// `ours`. A path is a file, so two sides that changed different files + /// of a directory merge cleanly. Without a base, every path either side + /// has counts as changed. Subtrees with equal ids are never read, so the + /// cost follows the size of the changes. + /// + /// # Errors + /// + /// Fails if a tree that differs cannot be read. + pub fn merge( + &self, + base: Option, + ours: TreeId, + theirs: TreeId, + ) -> Result { + self.merge_with(base, ours, theirs, OnConflict::KeepOurs) + } + + /// Merges like [`Store::merge`], but where both sides changed a text + /// file, the merged one holds both versions between conflict markers + /// naming the sides with `labels`, see [`Conflicts::mark`]. Binary files, + /// and paths a directory is on either side of, keep `ours`. + /// + /// # Errors + /// + /// Fails if a tree that differs, or a file that conflicts, cannot be + /// read. + pub fn merge_marked( + &self, + base: Option, + ours: TreeId, + theirs: TreeId, + labels: [&str; 2], + ) -> Result { + self.merge_with(base, ours, theirs, OnConflict::Mark(labels)) + } + + /// The entry a merge of merge bases has where they conflict: an id no + /// object has, so it equals neither side of a merge from it. A merge + /// never reads under it either: the bases may have had a directory + /// there, so any change at or under its path conflicts. + const UNKNOWN: TreeEntry = TreeEntry { + kind: EntryKind::File, + id: ObjectId::from_bytes([0; ObjectId::LEN]), + }; + + /// Merges like [`Store::merge`], but where both sides changed a path, + /// the merged tree has [`Store::UNKNOWN`]: how merge bases are merged + /// into one, so a merge from it conflicts there unless both sides + /// agree. + /// + /// # Errors + /// + /// Fails if a tree that differs cannot be read. + pub fn merge_bases( + &self, + base: Option, + ours: TreeId, + theirs: TreeId, + ) -> Result { + self.merge_with(base, ours, theirs, OnConflict::Unknown) + } + + fn merge_with( + &self, + base: Option, + ours: TreeId, + theirs: TreeId, + on_conflict: OnConflict<'_>, + ) -> Result { + let mut merge = Merge { + tree: ours, + trees: Vec::new(), + files: Vec::new(), + conflicts: Conflicts::default(), + }; + let merged = self.merge_dir( + &RepoPath::root(), + base, + Some(ours), + Some(theirs), + on_conflict, + &mut merge, + )?; + merge.tree = if let Some(tree) = merged { + tree + } else { + let empty = Tree::default().encode(); + let id = empty.id().into(); + merge.trees.push(empty); + id + }; + merge.trees.sort_unstable_by_key(EncodedObject::id); + merge.trees.dedup_by_key(|tree| tree.id()); + merge.files.sort_unstable_by_key(EncodedObject::id); + merge.files.dedup_by_key(|file| file.id()); + Ok(merge) + } + + /// Merges three versions of the directory `dir`, `None` standing for one + /// that doesn't exist, returning the merged one, or `None` if it's + /// empty. + fn merge_dir( + &self, + dir: &RepoPath, + base: Option, + ours: Option, + theirs: Option, + on_conflict: OnConflict<'_>, + merge: &mut Merge, + ) -> Result, StoreError> { + if ours == theirs || base == theirs { + return Ok(ours); + } + if base == ours { + return Ok(theirs); + } + let read = |id: Option| id.map(|id| self.get(id)).transpose(); + let (base, ours, theirs): (Tree, Tree, Tree) = ( + read(base)?.unwrap_or_default(), + read(ours)?.unwrap_or_default(), + read(theirs)?.unwrap_or_default(), + ); + let mut names: Vec<&EntryName> = [&base, &ours, &theirs] + .into_iter() + .flat_map(|tree| tree.iter().map(|(name, _)| name)) + .collect(); + names.sort_unstable(); + names.dedup(); + let mut merged = Tree::default(); + for name in names { + let [base, ours, theirs] = + [&base, &ours, &theirs].map(|tree| tree.get(name.as_str()).copied()); + let entry = if ours == theirs || base == theirs { + ours + } else if base == ours { + theirs + } else if base != Some(Self::UNKNOWN) + && ours.is_none_or(|ours| ours.kind.is_tree()) + && theirs.is_none_or(|theirs| theirs.kind.is_tree()) + { + // Both sides changed a directory, or one changed it and the + // other removed it: merge it file by file, unless what it was + // is unknown, which would take what one side removed for + // what the other added. + let subtree = |entry: Option| entry.and_then(TreeEntry::subtree); + self.merge_dir( + &dir.join(name), + subtree(base), + subtree(ours), + subtree(theirs), + on_conflict, + merge, + )? + .map(|tree| TreeEntry { + kind: EntryKind::Tree, + id: tree.into(), + }) + } else { + let written = match on_conflict { + OnConflict::KeepOurs => ours, + OnConflict::Mark(labels) => self.mark(ours, theirs, labels, merge)?, + OnConflict::Unknown => Some(Self::UNKNOWN), + }; + // Paths come in order, each once. + merge.conflicts.push(Conflicted { + path: dir.join(name), + written: written.map(|entry| entry.id), + }); + written + }; + if let Some(entry) = entry { + merged.insert(name.clone(), entry); + } + } + if merged.is_empty() { + return Ok(None); + } + let encoded = merged.encode(); + let id = encoded.id().into(); + merge.trees.push(encoded); + Ok(Some(id)) + } + + /// The entry of a file both sides changed, `ours` and `theirs`, that + /// holds both versions between conflict markers: a file on the side + /// that removed it is empty. Keeps `ours` if either side isn't a + /// regular file, or is binary or missing from the store. + fn mark( + &self, + ours: Option, + theirs: Option, + labels: [&str; 2], + merge: &mut Merge, + ) -> Result, StoreError> { + let content = |entry: Option| match entry { + None => Ok(Some(Vec::new())), + Some(entry) if matches!(entry.kind, EntryKind::File | EntryKind::Executable) => { + self.try_blob(entry.id) + } + Some(_) => Ok(None), + }; + let (Some(our_content), Some(their_content)) = (content(ours)?, content(theirs)?) else { + return Ok(ours); + }; + let Some(marked) = Conflicts::mark(&our_content, &their_content, labels) else { + return Ok(ours); + }; + let file = EncodedObject::blob(&marked); + let kind = ours.or(theirs).map_or(EntryKind::File, |entry| entry.kind); + let entry = TreeEntry { + kind, + id: file.id(), + }; + merge.files.push(file); + Ok(Some(entry)) + } +} + /// Looks up the entries of snapshots by path, reading each tree once, /// however many snapshots share it. #[derive(Debug)] @@ -271,6 +524,118 @@ pub(crate) mod tests { assert_eq!(store.diff(empty, old).unwrap().len(), 5); } + /// The files of `tree`, by path, with their contents' ids. + fn files(store: &Store, tree: TreeId) -> Vec<(String, ObjectId)> { + let empty = build(store, &[]); + store + .diff(empty, tree) + .unwrap() + .into_iter() + .map(|change| (change.path.as_str().to_owned(), change.new.unwrap().id)) + .collect() + } + + #[test] + fn merges_changes_to_different_files() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let base = build( + &store, + &[ + ("src/a", "a"), + ("src/b", "b"), + ("gone/x", "x"), + ("same", "s"), + ], + ); + // Ours edits a file and removes a directory; theirs edits another + // file of the same directory, adds one, and makes the same edit. + let ours = build(&store, &[("src/a", "a2"), ("src/b", "b"), ("same", "s2")]); + let theirs = build( + &store, + &[ + ("src/a", "a"), + ("src/b", "b2"), + ("gone/x", "x"), + ("new/deep", "n"), + ("same", "s2"), + ], + ); + let merge = store.merge(Some(base), ours, theirs).unwrap(); + assert_eq!(merge.conflicts, Conflicts::default()); + let expected = build( + &store, + &[ + ("src/a", "a2"), + ("src/b", "b2"), + ("new/deep", "n"), + ("same", "s2"), + ], + ); + assert_eq!(files(&store, merge.tree), files(&store, expected)); + assert_eq!(merge.tree, expected, "trees are canonical"); + assert!( + merge.trees.iter().any(|tree| tree.id() == expected.into()), + "the merged trees are returned to be written" + ); + + // Merging onto an unchanged base reads nothing and makes no tree. + let unchanged = store.merge(Some(base), ours, base).unwrap(); + assert_eq!((unchanged.tree, unchanged.trees), (ours, Vec::new())); + } + + #[test] + fn conflicts_keep_ours() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let base = build( + &store, + &[("edit", "1"), ("dir/removed", "r"), ("file", "f")], + ); + let ours = build( + &store, + &[("edit", "ours"), ("dir/removed", "r2"), ("file", "f")], + ); + let theirs = build( + &store, + &[("edit", "theirs"), ("file/now-dir", "d"), ("other", "o")], + ); + let merge = store.merge(Some(base), ours, theirs).unwrap(); + let paths: Vec<_> = merge + .conflicts + .paths() + .iter() + .map(ToString::to_string) + .collect(); + assert_eq!( + paths, + ["dir/removed", "edit"], + "both edited `edit`; ours edited what theirs removed" + ); + let expected = build( + &store, + &[ + ("edit", "ours"), + ("dir/removed", "r2"), + ("file/now-dir", "d"), + ("other", "o"), + ], + ); + assert_eq!(files(&store, merge.tree), files(&store, expected)); + + // Without a base, both sides added everything. + let unrelated = store.merge(None, ours, theirs).unwrap(); + let paths: Vec<_> = unrelated + .conflicts + .paths() + .iter() + .map(ToString::to_string) + .collect(); + assert_eq!(paths, ["edit", "file"]); + } + #[test] fn looks_entries_up_by_path() { let dir = tempfile::tempdir().unwrap(); diff --git a/apps/origin-src/src/workspace/annotate.rs b/apps/origin-src/src/workspace/annotate.rs index 210bd5a2..164a4972 100644 --- a/apps/origin-src/src/workspace/annotate.rs +++ b/apps/origin-src/src/workspace/annotate.rs @@ -93,7 +93,7 @@ impl Payload { impl Workspace { /// This node, which issues the statements core makes. - fn node_principal(&self) -> Principal { + pub(super) fn node_principal(&self) -> Principal { Principal::Node(self.config.node().clone()) } @@ -326,7 +326,7 @@ impl Workspace { let working_copy = self.refresh()?; let mut annotations = Annotations::open(&self.store)?; let revision = working_copy.find_revision(annotations.index(), rev)?; - let promoted = annotations.promote(revision, &self.node_principal())?; + let promoted = annotations.promote(RecordId::Revision(revision), &self.node_principal())?; if promoted.is_empty() { println!("Nothing to promote to revision {}", revision.short()); } diff --git a/apps/origin-src/src/workspace/land.rs b/apps/origin-src/src/workspace/land.rs new file mode 100644 index 00000000..a87fe7cb --- /dev/null +++ b/apps/origin-src/src/workspace/land.rs @@ -0,0 +1,334 @@ +//! The target commands: `land`, `target new` and `targets`, and starting a +//! change on a target with `new --on`. +//! +//! The queue runs in this process, as this node. After a landing, the +//! working copy follows its change: when it landed, or the target already +//! had it, the working copy starts a new change on the target's tip, so +//! later edits are made on what the target has; when the queue rebased +//! it, the working copy becomes the rebased revision; and when it sits on +//! a change that landed, it is rebased onto the tip, if that merges +//! cleanly. + +use origin_shared::{ + Base, ChangeId, EncodedObject, Record, RecordId, Revision, RevisionId, Source, TargetName, + TreeId, +}; + +use super::{WorkingCopy, Workspace, WorkspaceError}; +use crate::index::Index; +use crate::policy::Policy; +use crate::queue::{Blocked, Landing, Pending, Proposal, Queue}; + +impl Workspace { + /// `origin land [] [--onto ]`: lands a change, the + /// working copy's by default, with the changes below it the target + /// doesn't have yet, on the target its stack is on by default. + /// + /// # Errors + /// + /// Fails if the change cannot be found or cannot land, this node isn't + /// the target's authority, or the records cannot be read or written. + pub fn land( + &mut self, + change: Option<&str>, + onto: Option, + ) -> Result<(), WorkspaceError> { + let working_copy = self.refresh()?; + let index = Index::open(&self.store)?; + let change = working_copy.find_change(&index, change.unwrap_or("@"))?; + let onto = match onto { + Some(onto) => onto, + None => self.target_of(Self::single_head(&index, change)?)?, + }; + let key = self.config.node_key()?; + let landings = + Queue::new(&self.store, &key, self.config.node().clone()).land_change(change, &onto)?; + for landing in &landings { + self.print_landing(landing, &onto)?; + } + self.follow(&working_copy, &onto, &landings) + } + + /// `origin land --from --onto `: lands the tip of + /// another target. + /// + /// # Errors + /// + /// Fails if a target doesn't exist, this node isn't the authority of + /// `onto`, or the records cannot be read or written. + pub fn land_target(&self, from: &TargetName, onto: &TargetName) -> Result<(), WorkspaceError> { + let key = self.config.node_key()?; + let landing = + Queue::new(&self.store, &key, self.config.node().clone()).land_target(from, onto)?; + self.print_landing(&landing, onto) + } + + /// `origin land --status`: lists the landings that wait, and why. + /// + /// # Errors + /// + /// Fails if the queues cannot be read. + pub fn land_status(&self) -> Result<(), WorkspaceError> { + let pending = Queue::pending(&self.store)?; + if pending.is_empty() { + println!("No landing waits."); + } + for pending in &pending { + Self::print_pending(pending); + } + Ok(()) + } + + /// `origin target new --from `: creates a target at the + /// tip of another. + /// + /// # Errors + /// + /// Fails if the name is taken, `from` doesn't exist, this node wouldn't + /// be the target's authority, or the records cannot be written. + pub fn target_new(&self, name: &TargetName, from: &TargetName) -> Result<(), WorkspaceError> { + let key = self.config.node_key()?; + Queue::new(&self.store, &key, self.config.node().clone()).create_target(name, from)?; + let tip = self + .store + .head(from)? + .ok_or_else(|| WorkspaceError::UnknownTarget(from.clone()))?; + println!( + "Created target `{name}` at {}", + self.describe_acceptance(tip.acceptance)? + ); + Ok(()) + } + + /// `origin targets`: lists every target with its tip, its authority and + /// what landing on it requires. + /// + /// # Errors + /// + /// Fails if the targets or the policy cannot be read. + pub fn targets(&self) -> Result<(), WorkspaceError> { + let policy = Policy::load(&self.store)?; + let mut targets = self.store.targets()?; + targets.sort(); + for target in targets { + let Some(tip) = self.store.head(&target)? else { + continue; + }; + let authority = policy + .authority(&target) + .map_or_else(|| "no authority".to_owned(), |node| format!("node:{node}")); + println!( + "{target} #{} {} ({authority})", + tip.seq, + tip.acceptance.short() + ); + for requirement in policy.requirements(&target) { + println!(" requires {requirement}"); + } + } + Ok(()) + } + + /// `origin new [description] --on `: starts a change on the tip + /// of a target. + /// + /// # Errors + /// + /// Fails if the target doesn't exist, the working copy cannot be + /// snapshotted, or the change cannot be written or checked out. + pub fn new_on(&mut self, target: &TargetName, description: &str) -> Result<(), WorkspaceError> { + let working_copy = self.refresh()?; + self.start_on(target, description, working_copy.revision.snapshot) + } + + /// Starts a change described by `description` on the tip of `target`, + /// and checks it out over the files of `files`. + fn start_on( + &mut self, + target: &TargetName, + description: &str, + files: TreeId, + ) -> Result<(), WorkspaceError> { + let tip = self + .store + .head(target)? + .ok_or_else(|| WorkspaceError::UnknownTarget(target.clone()))?; + let description = EncodedObject::blob(description.as_bytes()); + let revision = Revision { + change: ChangeId::random().map_err(WorkspaceError::Random)?, + predecessors: Vec::new(), + base: Base::Acceptance(tip.acceptance), + deps: Vec::new(), + snapshot: self.store.get(tip.acceptance)?.body().state, + description: description.id().into(), + author: self.config.user(), + }; + let encoded = revision.encode(); + self.write(&[vec![description], vec![encoded.clone()]])?; + self.checkout(files, revision.snapshot)?; + self.point_at(encoded.id().into())?; + println!("Working copy: {}", self.headline(&revision)?); + println!("On top of: {}", self.describe_base(revision.base)?); + Ok(()) + } + + /// The target a revision's stack is on: that of the acceptance at its + /// bottom. + fn target_of(&self, revision: RevisionId) -> Result { + let mut base = self.store.get(revision)?.base; + loop { + match base { + Base::Revision(id) => base = self.store.get(id)?.base, + Base::Acceptance(id) => { + return self.target_name(self.store.get(id)?.body().target); + } + } + } + } + + /// Moves the working copy after `landings` on `onto`, as the module + /// explains. + fn follow( + &mut self, + working_copy: &WorkingCopy, + onto: &TargetName, + landings: &[Landing], + ) -> Result<(), WorkspaceError> { + let revision = &working_copy.revision; + let files = revision.snapshot; + // A change the target already had is as good as landed. + let landed = |change: ChangeId| { + landings.iter().any(|landing| match landing { + Landing::Landed { proposal, .. } | Landing::Unchanged(proposal) => { + *proposal == Proposal::Change(change) + } + Landing::Pending(_) => false, + }) + }; + if landed(revision.change) { + return self.start_on(onto, "", files); + } + for landing in landings { + if let Landing::Pending(Pending { + proposal: Proposal::Change(change), + subject: RecordId::Revision(rebased), + .. + }) = landing + && *change == revision.change + && *rebased != working_copy.id + { + let rebased = *rebased; + self.checkout(files, self.store.get(rebased)?.snapshot)?; + self.point_at(rebased)?; + println!( + "Working copy: {}", + self.headline(&self.store.get(rebased)?)? + ); + return Ok(()); + } + } + let Base::Revision(below) = revision.base else { + return Ok(()); + }; + let below = self.store.get(below)?; + if !landed(below.change) { + return Ok(()); + } + self.rebase_onto_tip(working_copy, below.snapshot, onto) + } + + /// Rebases the working copy, made on `base`, onto the tip of `onto`, if + /// that merges cleanly. + fn rebase_onto_tip( + &mut self, + working_copy: &WorkingCopy, + base: TreeId, + onto: &TargetName, + ) -> Result<(), WorkspaceError> { + let tip = self + .store + .head(onto)? + .ok_or_else(|| WorkspaceError::UnknownTarget(onto.clone()))?; + let state = self.store.get(tip.acceptance)?.body().state; + let current = &working_copy.revision; + let merge = self.store.merge(Some(base), current.snapshot, state)?; + if !merge.conflicts.is_empty() { + println!( + "The working copy stays where it is: it conflicts with `{onto}` in {}", + Self::list_paths(&merge.conflicts.paths()) + ); + return Ok(()); + } + let revision = Revision { + predecessors: vec![working_copy.id], + base: Base::Acceptance(tip.acceptance), + snapshot: merge.tree, + author: self.config.user(), + ..current.clone() + }; + let encoded = revision.encode(); + self.write(&[merge.trees, vec![encoded.clone()]])?; + self.checkout(current.snapshot, revision.snapshot)?; + self.point_at(encoded.id().into())?; + println!("Working copy: {}", self.headline(&revision)?); + println!("On top of: {}", self.describe_base(revision.base)?); + Ok(()) + } + + fn print_landing(&self, landing: &Landing, onto: &TargetName) -> Result<(), WorkspaceError> { + match landing { + Landing::Landed { + proposal, + source, + acceptance, + } => { + let landed = match source { + Source::Revision(revision) => format!(", revision {},", revision.short()), + Source::Target(_) => String::new(), + }; + println!( + "Landed {proposal}{landed} on {}", + self.describe_acceptance(acceptance.acceptance)? + ); + } + Landing::Unchanged(proposal) => { + println!("Landing {proposal} would change nothing on `{onto}`"); + } + Landing::Pending(pending) => Self::print_pending(pending), + } + Ok(()) + } + + fn print_pending(pending: &Pending) { + let Pending { + onto, + proposal, + subject, + blocked, + } = pending; + let retry = match proposal { + Proposal::Change(change) => format!("origin land {} --onto {onto}", change.short()), + Proposal::Target(target) => format!("origin land --from {target} --onto {onto}"), + }; + match blocked { + Blocked::Evidence(missing) => { + let missing: Vec<_> = missing.iter().map(ToString::to_string).collect(); + println!( + "The landing of {proposal} on `{onto}` waits for {}", + missing.join(", ") + ); + println!( + " about {}: once they're issued, run `{retry}`", + subject.describe() + ); + } + Blocked::Conflict { paths, .. } => { + println!( + "The landing of {proposal} on `{onto}` conflicts in {}", + Self::list_paths(paths) + ); + println!(" resolve it in a new revision, then run `{retry}`"); + } + } + } +} diff --git a/apps/origin-src/src/workspace/mod.rs b/apps/origin-src/src/workspace/mod.rs index ebe67256..0c123198 100644 --- a/apps/origin-src/src/workspace/mod.rs +++ b/apps/origin-src/src/workspace/mod.rs @@ -14,9 +14,12 @@ //! the files changed, they become a new revision of the change, which //! evolves from the previous one. Records are written before anything //! refers to them: the snapshot's objects, then the revision, then the -//! `workspace` file naming it. +//! `workspace` file naming it. Only the statements a new revision needs, +//! such as its unresolved conflicts, come before it, so it's never found +//! without them. mod annotate; +mod land; pub use annotate::{Annotated, Payload}; @@ -26,15 +29,19 @@ use std::path::{self, Path, PathBuf}; use origin_platform::Native; use origin_shared::{ - AcceptanceBody, AcceptanceId, Base, BlobId, ChangeId, EncodedObject, EntryKind, EntryName, - Kind, NodePublicKey, Principal, Record, RecordId, Revision, RevisionId, Source, Statement, - StatementId, TargetId, TargetName, Tree, TreeEntry, TreeId, + AcceptanceId, Base, BlobId, ChangeId, EncodedObject, Kind, NodePublicKey, Principal, Record, + RecordId, RepoPath, Revision, RevisionId, Source, Statement, StatementId, TargetId, TargetName, + Tree, TreeId, }; -use crate::annotations::AnnotateError; +use crate::ancestry::Ancestry; +use crate::annotations::{AnnotateError, Annotations}; use crate::config::{ConfigError, UserConfig}; +use crate::conflict::Conflicts; use crate::index::{FindChangeError, FindError, Index}; use crate::output::FileDiff; +use crate::policy::PolicyError; +use crate::queue::{LandError, Queue}; use crate::store::{Store, StoreError}; use crate::worktree::{SnapshotError, StatCache, Worktree}; @@ -58,6 +65,8 @@ struct WorkingCopy { /// The revision's record, if recording the draft made a new revision, /// which is not written yet. unsaved: Option, + /// The unresolved conflicts of the revision that the files still have. + conflicts: Conflicts, } impl WorkingCopy { @@ -94,32 +103,10 @@ impl Workspace { let store = Store::new(meta, Native::default()); store.init()?; - let (policy, policy_tree) = Self::policy(&config, &key.public()); + let policy = Self::policy(&config, &key.public()); + let main = Queue::new(&store, &key, config.node().clone()).init(&policy)?; let empty_tree = Tree::default().encode(); let empty_description = EncodedObject::blob(b""); - let genesis = |target: &TargetName, state: &EncodedObject, policy| AcceptanceBody { - target: target.id(), - seq: 0, - prev: None, - state: state.id().into(), - landed: Vec::new(), - evidence: Vec::new(), - policy, - }; - store.write_all(&[ - policy, - policy_tree.clone(), - empty_tree.clone(), - empty_description.clone(), - ])?; - let policy = store.append( - &TargetName::POLICY, - &genesis(&TargetName::POLICY, &policy_tree, None).sign(&key), - )?; - let main = store.append( - &TargetName::MAIN, - &genesis(&TargetName::MAIN, &empty_tree, Some(policy)).sign(&key), - )?; let revision = Revision { change: ChangeId::random().map_err(WorkspaceError::Random)?, @@ -137,7 +124,10 @@ impl Workspace { config, revision: encoded.id().into(), }; - workspace.save(&[vec![encoded]], workspace.revision)?; + workspace.save( + &[vec![empty_description], vec![encoded]], + workspace.revision, + )?; println!( "Initialized a repository in {}: node:{} is the authority of `policy` and `main`", workspace.root.display(), @@ -147,34 +137,30 @@ impl Workspace { Ok(workspace) } - /// The first `policy.toml`, and the snapshot holding it: this node is - /// the authority of `policy` and `main`, and neither requires anything, - /// so a solo project lands instantly. - fn policy(config: &UserConfig, key: &NodePublicKey) -> (EncodedObject, EncodedObject) { + /// The first `policy.toml`: this node is the authority of `policy` and + /// `main`, and neither requires anything, so a solo project lands + /// instantly. + fn policy(config: &UserConfig, key: &NodePublicKey) -> String { let node = config.node(); - let policy = format!( + format!( "# Who may land what. Changing it is a landing on `policy`.\n\ \n\ + # Each node's public key.\n\ [nodes]\n\ \"{node}\" = \"{key}\"\n\ \n\ + # Groups of users and nodes, for requirements:\n\ + # [groups]\n\ + # runners = [\"node:ci-1\"]\n\ + \n\ + # The node ordering each target, and the statements landing needs.\n\ [targets.policy]\n\ authority = \"node:{node}\"\n\ \n\ [targets.main]\n\ - authority = \"node:{node}\"\n" - ); - let policy = EncodedObject::blob(policy.as_bytes()); - let tree: Tree = [( - EntryName::new("policy.toml").expect("a valid name"), - TreeEntry { - kind: EntryKind::File, - id: policy.id(), - }, - )] - .into_iter() - .collect(); - (policy, tree.encode()) + authority = \"node:{node}\"\n\ + # require = [{{ kind = \"ci.test\", verdict = \"pass\", issuer = \"group:runners\" }}]\n" + ) } /// Opens the working copy containing `start`. @@ -225,11 +211,15 @@ impl Workspace { .save(&path) .map_err(|source| WorkspaceError::io(&path, source))?; } + let conflicts = self + .conflicts(self.revision, &revision)? + .remaining(&self.store, files)?; Ok(WorkingCopy { id: self.revision, revision, files, unsaved: None, + conflicts, }) } @@ -247,6 +237,10 @@ impl Workspace { ..working_copy.revision }; let encoded = revision.encode(); + if !working_copy.conflicts.is_empty() { + // Now, so the revision never exists without them. + self.write(&self.revision_layers(Vec::new(), &encoded, &working_copy.conflicts))?; + } Ok(WorkingCopy { id: encoded.id().into(), revision, @@ -255,6 +249,46 @@ impl Workspace { }) } + /// The unresolved conflicts of `revision`, `id`. + fn conflicts(&self, id: RevisionId, revision: &Revision) -> Result { + let annotations = Annotations::open(&self.store)?; + let claims = annotations.claims(RecordId::Revision(id))?; + Ok(Conflicts::of(&self.store, &claims, revision)?) + } + + /// The unresolved conflicts a new change on `base` starts with, from + /// the files of the revision it's on, if it's on one. + fn base_conflicts(&self, base: Base) -> Result { + match base { + Base::Revision(id) => self.conflicts(id, &self.store.get(id)?), + Base::Acceptance(_) => Ok(Conflicts::default()), + } + } + + /// The layers that write `revision`, a new one: `first`, objects it + /// refers to, then the statements published with it, then the revision + /// itself. The statements are a `core.conflict` about it listing + /// `conflicts`, its unresolved ones, if it has any (see + /// [`conflict`](crate::conflict)), and any a caller adds to the second + /// layer. Whoever finds the revision finds them: an error or a crash + /// never publishes it without them, as one batch could, publishing some + /// of its files. + fn revision_layers( + &self, + mut first: Vec, + revision: &EncodedObject, + conflicts: &Conflicts, + ) -> Vec> { + let mut statements = Vec::new(); + if !conflicts.is_empty() { + let subject = RecordId::Revision(revision.id().into()); + let (payload, statement) = conflicts.statement(subject, self.config.user(), None); + first.push(payload); + statements.push(statement.encode()); + } + vec![first, statements, vec![revision.clone()]] + } + /// Snapshots the working copy and saves its draft as a new revision if /// the files changed, for the commands that build on it. fn refresh(&mut self) -> Result { @@ -321,6 +355,9 @@ impl Workspace { ); println!("On top of: {}", self.describe_base(revision.base)?); Self::print_divergence(&index, revision.change); + if !working_copy.conflicts.is_empty() { + Self::print_conflicts(&working_copy.conflicts); + } let changes = self .store .diff(self.base_tree(revision.base)?, working_copy.files)?; @@ -434,7 +471,9 @@ impl Workspace { let id = encoded.id().into(); let recorded = working_copy.unsaved.is_some(); let first = working_copy.unsaved.into_iter().chain([description]); - self.save(&[first.collect(), vec![encoded]], id)?; + // It starts with the files of the one below, and their conflicts. + let layers = self.revision_layers(first.collect(), &encoded, &working_copy.conflicts); + self.save(&layers, id)?; // Edits recorded on a revision that is no longer its change's head // make that change diverge. if recorded { @@ -464,11 +503,43 @@ impl Workspace { let encoded = revision.encode(); let id = encoded.id().into(); let first = working_copy.unsaved.into_iter().chain([description]); - self.save(&[first.collect(), vec![encoded]], id)?; + let layers = self.revision_layers(first.collect(), &encoded, &working_copy.conflicts); + self.save(&layers, id)?; println!("Working copy: {}", self.headline(&revision)?); Ok(()) } + /// `origin resolve ...`: accepts the files at `paths` as they + /// are, resolving their conflicts, with a new revision of the change + /// that keeps the others. Paths without one are skipped. + /// + /// # Errors + /// + /// Fails if the working copy cannot be snapshotted or the revision + /// cannot be written. + pub fn resolve(&mut self, paths: &[RepoPath]) -> Result<(), WorkspaceError> { + let mut working_copy = self.refresh()?; + let resolved = working_copy.conflicts.resolve(paths); + if resolved.is_empty() { + println!("No conflict to resolve there."); + return Ok(()); + } + let revision = Revision { + predecessors: vec![working_copy.id], + author: self.config.user(), + ..working_copy.revision + }; + let encoded = revision.encode(); + let id = encoded.id().into(); + let layers = self.revision_layers(Vec::new(), &encoded, &working_copy.conflicts); + self.save(&layers, id)?; + println!("Resolved {}", Self::list_paths(&resolved)); + if !working_copy.conflicts.is_empty() { + Self::print_conflicts(&working_copy.conflicts); + } + Ok(()) + } + /// `origin edit `: switches the working copy to the change's /// head. /// @@ -546,7 +617,11 @@ impl Workspace { author: self.config.user(), }; let encoded = revision.encode(); - self.write(&[vec![reason, empty], vec![statement, encoded.clone()]])?; + let conflicts = self.base_conflicts(revision.base)?; + let mut layers = self.revision_layers(vec![reason, empty], &encoded, &conflicts); + // Published before the revision that replaces the change. + layers[1].push(statement); + self.write(&layers)?; self.checkout(working_copy.revision.snapshot, revision.snapshot)?; self.point_at(encoded.id().into())?; println!("Abandoned change {}", change.short()); @@ -556,9 +631,14 @@ impl Workspace { } /// `origin converge `: ends a change's divergence with a - /// revision evolving from all its heads. It keeps the files and - /// description of the working copy, if it is one of the heads, or else - /// of the newest head. + /// revision evolving from all its heads. It merges their files, file by + /// file, with the changes each made since the latest revision they all + /// evolve from. It keeps the base and the description of the working + /// copy, if it is one of the heads, or else of the newest head. Where + /// heads changed the same text file, it holds each version between + /// conflict markers, and a `core.conflict` statement lists the files, + /// so the revision doesn't land until a snapshot without the markers: + /// see [`conflict`](crate::conflict). /// /// # Errors /// @@ -577,28 +657,87 @@ impl Workspace { } else { heads[heads.len() - 1] }; + let kept_revision: Revision = self.store.get(kept)?; + let base = self.merge_base(&heads)?; + let mut snapshot = kept_revision.snapshot; + let mut merged = Conflicts::default(); + for (merged_heads, &head) in heads.iter().filter(|&&head| head != kept).enumerate() { + // Past the first merge, our side holds every head merged so far. + let ours = if merged_heads == 0 { + kept.short() + } else { + "merged heads".to_owned() + }; + let merge = self.store.merge_marked( + base, + snapshot, + self.store.get(head)?.snapshot, + [&ours, &head.short()], + )?; + // The next merge reads this one's trees. + self.write(&[merge.files.into_iter().chain(merge.trees).collect()])?; + snapshot = merge.tree; + merged = merged.with(merge.conflicts); + } + // The heads' own, where the merge kept their side. + let mut conflicts = Conflicts::default(); + for &head in &heads { + let unresolved = self.conflicts(head, &self.store.get(head)?)?; + conflicts = conflicts.with(unresolved.remaining(&self.store, snapshot)?); + } + let conflicts = conflicts.with(merged); let revision = Revision { predecessors: heads.clone(), + snapshot, author: self.config.user(), - ..self.store.get(kept)? + ..kept_revision }; let encoded = revision.encode(); let id = encoded.id().into(); - self.write(&[vec![encoded]])?; + self.write(&self.revision_layers(Vec::new(), &encoded, &conflicts))?; if change == working_copy.revision.change { self.checkout(working_copy.revision.snapshot, revision.snapshot)?; self.point_at(id)?; } println!( - "Converged change {}: revision {} evolves from {}, with the files of {}", + "Converged change {}: revision {} evolves from {}, merging their files", change.short(), id.short(), Self::list(&heads), - kept.short() ); + if !conflicts.is_empty() { + Self::print_conflicts(&conflicts); + } Ok(()) } + /// Lists a revision's unresolved conflicts, and how to resolve them. + fn print_conflicts(conflicts: &Conflicts) { + println!( + "Conflicts: {} hold both versions between conflict markers, or one side for a \ + binary file. Edit them, or run `origin resolve ...` to accept them as they \ + are.", + Self::list_paths(&conflicts.paths()) + ); + } + + /// `paths`, separated by commas. + fn list_paths(paths: &[RepoPath]) -> String { + let paths: Vec<_> = paths.iter().map(ToString::to_string).collect(); + paths.join(", ") + } + + /// The snapshot to merge `heads`, revisions of one change, from: that of + /// the newest revision they all evolve from, or the + /// [merge](Ancestry::merge_base) of several, if each lacks another. + fn merge_base(&self, heads: &[RevisionId]) -> Result, WorkspaceError> { + let ancestry = Ancestry::walk(heads, |id| { + let revision: Revision = self.store.get(id)?; + Ok::<_, StoreError>((revision.predecessors, revision.snapshot)) + })?; + Ok(ancestry.merge_base(&self.store, heads)?) + } + /// `origin log []`: prints the working copy's stack of changes /// down to the target it's on, then the target's acceptances, newest /// first. With a target, prints that target's acceptances instead, @@ -951,6 +1090,12 @@ pub enum WorkspaceError { /// A statement cannot be issued. #[error(transparent)] Annotate(#[from] AnnotateError), + /// The policy cannot be read. + #[error(transparent)] + Policy(#[from] PolicyError), + /// A landing failed. + #[error(transparent)] + Land(#[from] LandError), } impl WorkspaceError { diff --git a/apps/origin-src/tests/cli.rs b/apps/origin-src/tests/cli.rs index 219b9b43..34fe0e8f 100644 --- a/apps/origin-src/tests/cli.rs +++ b/apps/origin-src/tests/cli.rs @@ -260,19 +260,118 @@ fn divergence_shows_until_converged() { converged.starts_with(&format!("Converged change {change}")), "{converged}" ); - assert_eq!( - repo.read("file"), - "right\n", - "the working copy's files are kept" + assert!( + converged.contains("Conflicts: file hold both versions"), + "{converged}" ); - assert!(!repo.ok(&["status"]).contains("divergent")); + // Both versions, the working copy's first, between markers. + let file = repo.read("file"); + assert!(file.starts_with("<<<<<<< "), "{file}"); + assert!(file.contains("\nright\n=======\nleft\n>>>>>>> "), "{file}"); + let status = repo.ok(&["status"]); + assert!(!status.contains("divergent"), "{status}"); + assert!(status.contains("Conflicts: file hold"), "{status}"); assert!(repo.ok(&["evolog"]).contains(" converges ")); + + // It doesn't land until the file changes: describing it keeps the + // conflict, an edit resolves it, whatever the file then holds. + repo.ok(&["describe", "Shared work, converged"]); + let refused = repo.fails(&["land"]); + assert!(refused.contains("unresolved conflicts"), "{refused}"); + assert!(refused.contains("edit file"), "{refused}"); + repo.write("file", "Heading\n=======\nboth\n"); + assert!(!repo.ok(&["status"]).contains("Conflicts")); + assert!(repo.ok(&["land"]).starts_with("Landed change")); assert!( repo.fails(&["converge", &change]) .contains("is not divergent") ); } +#[test] +fn binary_conflicts_resolve_only_when_accepted() { + let repo = Repo::new(); + repo.write("image", "base\0"); + repo.write("text", "base\n"); + repo.ok(&["describe", "Shared work"]); + let change = repo.change(); + let pointer = repo.path(".origin/workspace"); + let started = fs::read_to_string(&pointer).unwrap(); + repo.write("image", "left\0"); + repo.write("text", "left\n"); + repo.ok(&["describe", "Shared work"]); + fs::write(&pointer, started).unwrap(); + repo.write("image", "right\0"); + repo.write("text", "right\n"); + repo.ok(&["converge", &change]); + assert_eq!(repo.read("image"), "right\0", "the working copy's side"); + + // An edit elsewhere leaves it unresolved, even without markers. + repo.write("unrelated", "docs\n"); + assert!(repo.fails(&["land"]).contains("unresolved conflicts")); + let resolved = repo.ok(&["resolve", "image", "unrelated"]); + assert!(resolved.starts_with("Resolved image\n"), "{resolved}"); + assert!(resolved.contains("Conflicts: text hold"), "{resolved}"); + let status = repo.ok(&["status"]); + assert!(status.contains("\nConflicts: text hold"), "{status}"); + repo.ok(&["resolve", "text"]); + assert!(repo.ok(&["land"]).starts_with("Landed change")); +} + +#[test] +fn a_revision_is_published_only_after_its_conflicts() { + use origin_shared::{EncodedObject, Record, RecordId, Revision, RevisionId}; + use origin_src::{annotations::Annotations, conflict::Conflicts, store::Store}; + + let repo = Repo::new(); + repo.write("file", "base\n"); + repo.ok(&["describe", "Shared work"]); + let change = repo.change(); + let pointer = repo.path(".origin/workspace"); + let started = fs::read_to_string(&pointer).unwrap(); + repo.write("file", "left\n"); + repo.ok(&["describe", "Shared work"]); + fs::write(&pointer, started).unwrap(); + repo.write("file", "right\n"); + repo.ok(&["converge", &change]); + + // The `core.conflict` statement `describe` issues about its revision. + let store = Store::new(repo.path(".origin"), origin_platform::Native::default()); + let converged: RevisionId = fs::read_to_string(&pointer) + .unwrap() + .trim() + .parse() + .unwrap(); + let revision: Revision = store.get(converged).unwrap(); + let claims = Annotations::open(&store) + .unwrap() + .claims(RecordId::Revision(converged)) + .unwrap(); + let conflicts = Conflicts::of(&store, &claims, &revision).unwrap(); + let issuer = revision.author.clone(); + let described = Revision { + predecessors: vec![converged], + description: EncodedObject::blob(b"Described").id().into(), + ..revision + } + .encode(); + let subject = RecordId::Revision(described.id().into()); + let (_, statement) = conflicts.statement(subject, issuer, None); + let hex = statement.encode().id().to_hex(); + + // Writing it fails: its directory is read-only. + let dir = repo.path(&format!(".origin/objects/loose/{}", &hex[..2])); + fs::create_dir_all(&dir).unwrap(); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).unwrap(); + repo.fails(&["describe", "Described"]); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).unwrap(); + + // So the revision, which would hide the conflict, wasn't written. + assert!(!store.contains(described.id()).unwrap()); + let refused = repo.fails(&["land"]); + assert!(refused.contains("unresolved conflicts"), "{refused}"); +} + #[test] fn abandoning_the_current_change_restores_its_base() { let repo = Repo::new(); @@ -823,3 +922,272 @@ fn statements_are_promoted_where_their_inputs_are_unchanged() { ); assert!(repo.fails(&["annotate", "@", "ci.lint"]).contains("--pass")); } + +#[test] +fn landing_moves_the_working_copy_onto_the_target() { + let repo = Repo::new(); + repo.write("a", "a\n"); + repo.ok(&["describe", "Add a"]); + let first = repo.change(); + repo.ok(&["new", "Add b"]); + let second = repo.change(); + repo.write("b", "b\n"); + repo.ok(&["new"]); + let empty = repo.change(); + + // The stack lands from the bottom; the empty working copy has nothing + // to land, and moves onto the new tip. + let land = repo.ok(&["land"]); + let lines: Vec<_> = land.lines().collect(); + assert!( + lines[0].starts_with(&format!("Landed change {first}, revision ")), + "{land}" + ); + assert!(lines[0].ends_with(", on main #1"), "{land}"); + assert!( + lines[1].starts_with(&format!("Landed change {second}, ")), + "{land}" + ); + assert_eq!( + lines[2], + format!("Landing change {empty} would change nothing on `main`") + ); + assert!( + repo.ok(&["status"]) + .contains("On top of: main #2\nNo changes.\n") + ); + + // Landing the working copy's change starts a new one on the tip. + repo.write("c", "c\n"); + let land = repo.ok(&["land"]); + assert!(land.contains(" on main #3\nWorking copy: "), "{land}"); + assert_ne!(repo.change(), empty); + assert!( + repo.ok(&["status"]) + .ends_with("On top of: main #3\nNo changes.\n") + ); + assert_eq!(repo.read("c"), "c\n"); + + let targets = repo.ok(&["targets"]); + assert!(targets.starts_with("main #3 "), "{targets}"); + assert!(targets.contains("(node:ada)\npolicy #0 "), "{targets}"); +} + +#[test] +fn a_change_the_target_already_has_moves_onto_its_tip() { + let repo = Repo::new(); + repo.write("f", "f\n"); + let ours = repo.change(); + repo.ok(&["new", "--on", "main", "The same file"]); + repo.write("f", "f\n"); + repo.ok(&["land"]); + repo.ok(&["edit", &ours]); + let land = repo.ok(&["land"]); + assert!(land.contains(" would change nothing on `main`\n"), "{land}"); + + // Deleting the file is then a change to what `main` has. + fs::remove_file(repo.path("f")).unwrap(); + let land = repo.ok(&["land"]); + assert!(land.starts_with("Landed change "), "{land}"); + assert!(!repo.path("f").exists()); +} + +#[test] +fn targets_fork_and_land_on_each_other() { + let repo = Repo::new(); + repo.ok(&["target", "new", "overhaul", "--from", "main"]); + assert!( + repo.fails(&["target", "new", "overhaul", "--from", "main"]) + .contains("already named `overhaul`") + ); + repo.ok(&["new", "--on", "overhaul", "Rework"]); + repo.write("rework", "new\n"); + let land = repo.ok(&["land"]); + assert!( + land.contains(" on overhaul #1\n"), + "lands where it's made: {land}" + ); + + let land = repo.ok(&["land", "--from", "overhaul", "--onto", "main"]); + assert_eq!(land, "Landed `overhaul` on main #1\n"); + assert!(repo.ok(&["log", "main"]).starts_with("◆ main #1 ")); + assert!(repo.ok(&["log", "main"]).contains("\n overhaul #1\n")); +} + +/// Lands a policy that makes `main` require `ci.test` to pass, from the +/// runners, ada among them. +fn require_tests(repo: &Repo) { + repo.ok(&["new", "--on", "policy", "Require tests"]); + let policy = repo + .read("policy.toml") + .replace("# [groups]", "[groups]") + .replace( + "# runners = [\"node:ci-1\"]", + "runners = [\"node:ci-1\", \"user:ada\"]", + ) + .replace("# require", "require"); + repo.write("policy.toml", &policy); + assert!(repo.ok(&["land"]).contains(" on policy #1\n")); +} + +#[test] +fn results_there_count_over_promoted_ones() { + let repo = Repo::new(); + require_tests(&repo); + repo.ok(&["new", "--on", "main", "Library"]); + repo.write("src/lib.rs", "fn a() {}\n"); + repo.ok(&["annotate", "@", "ci.test", "--pass", "--inputs", "src"]); + assert!(repo.ok(&["land"]).contains(" on main #1\n")); + // Each change below keeps `src`, so the pass is promoted to it. + let promoted = |description: &str| { + repo.ok(&["new", "--on", "main", description]); + repo.write(&format!("{description}.md"), "docs\n"); + assert!(repo.ok(&["promote"]).contains("Promoted ci.test pass")); + }; + let waits = || { + let landing = repo.ok(&["land"]); + assert!(landing.starts_with("The landing of change "), "{landing}"); + }; + // A pass there, then taken back, takes the promoted one with it. + promoted("withdrawn"); + repo.ok(&["annotate", "@", "ci.test", "--pass", "--inputs", "src"]); + repo.ok(&["annotate", "@", "ci.test", "--withdraw"]); + waits(); + // A failure there wins over the promoted pass. + promoted("failed"); + repo.ok(&["annotate", "@", "ci.test", "--fail", "--inputs", "src"]); + waits(); +} + +#[test] +fn landings_wait_for_the_evidence_policy_requires() { + let repo = Repo::new(); + require_tests(&repo); + assert!( + repo.ok(&["targets"]) + .contains(" requires ci.test pass from group:runners\n") + ); + + repo.ok(&["new", "--on", "main", "Feature"]); + let change = repo.change(); + repo.write("src/lib.rs", "fn a() {}\n"); + let waiting = repo.ok(&["land"]); + assert!( + waiting.starts_with(&format!( + "The landing of change {change} on `main` waits for ci.test pass from group:runners\n" + )), + "{waiting}" + ); + assert_eq!(repo.ok(&["land", "--status"]), waiting); + // A result withdrawn is no evidence. + repo.ok(&["annotate", "@", "ci.test", "--pass", "--inputs", "src"]); + repo.ok(&["annotate", "@", "ci.test", "--withdraw"]); + assert_eq!(repo.ok(&["land"]), waiting); + repo.ok(&["annotate", "@", "ci.test", "--pass", "--inputs", "src"]); + assert!(repo.ok(&["land"]).contains(" on main #1\n")); + assert_eq!(repo.ok(&["land", "--status"]), "No landing waits.\n"); + + // Only the authority lands: here, after a policy edit hands `main` to + // another node. + repo.ok(&["new", "--on", "policy"]); + let policy = repo.read("policy.toml").replace( + "[targets.main]\nauthority = \"node:ada\"", + "[targets.main]\nauthority = \"node:hosted\"", + ); + // Another node's key: any valid one will do. + let key = policy + .lines() + .find_map(|line| line.strip_prefix("\"ada\" = ")) + .unwrap(); + repo.write( + "policy.toml", + &policy.replace("[nodes]\n", &format!("[nodes]\nhosted = {key}\n")), + ); + repo.ok(&["land"]); + repo.ok(&["new", "--on", "main"]); + repo.write("x", "x\n"); + assert!( + repo.fails(&["land"]) + .contains("node:hosted is the authority of `main`") + ); +} + +#[test] +fn conflicting_landings_are_flagged_and_wait() { + let repo = Repo::new(); + repo.ok(&["describe", "One"]); + repo.write("shared", "one\n"); + let one = repo.change(); + repo.ok(&["new", "--on", "main", "Two"]); + repo.write("shared", "two\n"); + let two = repo.change(); + repo.ok(&["land", &one]); + + let conflict = repo.ok(&["land"]); + assert!( + conflict.starts_with(&format!( + "The landing of change {two} on `main` conflicts in shared\n" + )), + "{conflict}" + ); + assert!( + repo.ok(&["statements"]) + .contains(" core.conflict by node:ada\n") + ); + assert_eq!(repo.ok(&["land", "--status"]), conflict); +} + +#[test] +fn converge_merges_the_files_of_every_head() { + let repo = Repo::new(); + for path in ["a", "b", "c"] { + repo.write(path, "base\n"); + } + repo.ok(&["describe", "Shared work"]); + let change = repo.change(); + // Three agents revise the same revision, each changing its own file. + let pointer = repo.path(".origin/workspace"); + let started = fs::read_to_string(&pointer).unwrap(); + for changed in ["a", "b", "c"] { + fs::write(&pointer, &started).unwrap(); + for path in ["a", "b", "c"] { + repo.write( + path, + if path == changed { + "changed\n" + } else { + "base\n" + }, + ); + } + repo.ok(&["describe", "Shared work"]); + } + // Merging a third head reads the tree merging the first two. + let converged = repo.ok(&["converge", &change]); + assert!( + converged.ends_with(", merging their files\n"), + "{converged}" + ); + for path in ["a", "b", "c"] { + assert_eq!(repo.read(path), "changed\n", "{path}"); + } +} + +#[test] +fn converge_labels_the_side_holding_merged_heads() { + let repo = Repo::new(); + repo.write("file", "base\n"); + repo.ok(&["describe", "Shared work"]); + let change = repo.change(); + let pointer = repo.path(".origin/workspace"); + let started = fs::read_to_string(&pointer).unwrap(); + for side in ["one", "two", "three"] { + fs::write(&pointer, &started).unwrap(); + repo.write("file", &format!("{side}\n")); + repo.ok(&["describe", "Shared work"]); + } + repo.ok(&["converge", &change]); + // The third head merges into both versions of the first two. + let file = repo.read("file"); + assert!(file.starts_with("<<<<<<< merged heads\n<<<<<<< "), "{file}"); +} diff --git a/crates/origin-shared/src/object/acceptance.rs b/crates/origin-shared/src/object/acceptance.rs index 95ef6464..ecc82997 100644 --- a/crates/origin-shared/src/object/acceptance.rs +++ b/crates/origin-shared/src/object/acceptance.rs @@ -1,10 +1,11 @@ use std::fmt; +use std::str::FromStr; use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; use super::{DecodeError, Field, ObjectKind, Reader, Record, fields, tagged}; use crate::id::hex; -use crate::{AcceptanceId, RevisionId, StatementId, TargetId, TreeId}; +use crate::{AcceptanceId, ParseIdError, RevisionId, StatementId, TargetId, TreeId}; pub use ed25519_dalek::Signature; @@ -178,6 +179,21 @@ impl fmt::Display for NodePublicKey { } } +/// Parses the hex [`NodePublicKey`] displays as. +impl FromStr for NodePublicKey { + type Err = ParseIdError; + + fn from_str(input: &str) -> Result { + hex::decode(input) + .and_then(|bytes| VerifyingKey::from_bytes(&bytes).ok()) + .map(Self) + .ok_or_else(|| ParseIdError { + what: "node public key", + input: input.to_owned(), + }) + } +} + /// A signature that the claimed signer did not make. #[derive(Debug, thiserror::Error)] #[error("the signature does not match the signer's key")] diff --git a/crates/origin-shared/src/object/mod.rs b/crates/origin-shared/src/object/mod.rs index fe3742b3..4567e347 100644 --- a/crates/origin-shared/src/object/mod.rs +++ b/crates/origin-shared/src/object/mod.rs @@ -396,6 +396,9 @@ pub(crate) mod tests { acceptance.verify(&sample_key().public()).unwrap(); let other = NodeKey::from_bytes(&[1; 32]); assert!(acceptance.verify(&other.public()).is_err()); + let public = sample_key().public(); + assert_eq!(public.to_string().parse::().unwrap(), public); + assert!("ab".parse::().is_err()); // Any change to the body breaks the signature: here the sequence // number, right after the kind and the target id. diff --git a/crates/origin-shared/src/object/statement.rs b/crates/origin-shared/src/object/statement.rs index 616abbe2..aa171e9c 100644 --- a/crates/origin-shared/src/object/statement.rs +++ b/crates/origin-shared/src/object/statement.rs @@ -1,11 +1,13 @@ use std::borrow::Cow; use std::collections::HashSet; use std::fmt; +use std::str::FromStr; use super::{DecodeError, Field, ObjectKind, Reader, Record, fields, tagged}; use crate::name::names; use crate::{ - AcceptanceId, BlobId, Digest, Principal, RepoPath, RevisionId, StatementId, TreeEntry, TreeId, + AcceptanceId, BlobId, Digest, InvalidName, Principal, RepoPath, RevisionId, StatementId, + TreeEntry, TreeId, }; /// An attributable claim about another record: a check result, a review, a @@ -210,6 +212,23 @@ impl fmt::Display for Verdict { } } +/// Parses the names [`Verdict`] displays as: `pass`, `fail` and `neutral`. +impl FromStr for Verdict { + type Err = InvalidName; + + fn from_str(input: &str) -> Result { + match input { + "pass" => Ok(Self::Pass), + "fail" => Ok(Self::Fail), + "neutral" => Ok(Self::Neutral), + _ => Err(InvalidName { + what: "verdict", + name: input.to_owned(), + }), + } + } +} + impl Field for Verdict { fn write(&self, out: &mut Vec) { out.push(*self as u8); @@ -402,4 +421,12 @@ mod tests { } assert!(!Kind::new("corex.test").unwrap().is_reserved()); } + + #[test] + fn verdicts_parse_as_they_display() { + for verdict in [Verdict::Pass, Verdict::Fail, Verdict::Neutral] { + assert_eq!(verdict.to_string().parse::().unwrap(), verdict); + } + assert!("Pass".parse::().is_err()); + } } diff --git a/docs/design.md b/docs/design.md index d8958778..5eb3d4f2 100644 --- a/docs/design.md +++ b/docs/design.md @@ -379,7 +379,11 @@ and stops at the first one that waits. several, as git's recursive strategy does. The merge is checked as a revision on the tip, authored by the queue's node, so results about the same merge on another target never count. - - A landing that changes nothing is skipped. + - A landing that changes nothing is skipped: the target already has it. + But a merge of another target's tip that changes nothing still lands, + as an acceptance of the same state that needs no evidence, so later + merges start from it, unless either tip is already in the other's + history: then neither has anything the other lacks. 3. **On conflict,** issue a `core.conflict` statement and keep the landing blocked. Nothing fails, and landing again retries it. 4. **Check the policy** in force *now*: