From df1ef36218940a2066b86d1693558cfc7c6a8f84 Mon Sep 17 00:00:00 2001 From: piquel Date: Thu, 15 Oct 2026 19:20:52 +0200 Subject: [PATCH] feat: add targets, policy and the landing queue Changes could be made and annotated, but nothing moved a target past `init`. The landing queue now lands them: `origin land` rebases a change and the changes below it onto the target's tip with a path-level three-way merge, checks the statements `policy.toml` requires against the policy in force (promoting results whose input digest still matches), and appends an acceptance signed by the target's authority, one per change. Only checked claims count as evidence, a promotion as its original's issuer. Conflicts issue `core.conflict`, and landings without evidence wait in a durable, enumerable queue until they're retried; a stack that waits is kept whole. Divergent changes, those abandoned by their creator and revisions with unresolved conflicts are refused, and only the authority lands. A landing on `policy` is governed by the acceptance before it, never by the policy it lands (`Policy::governing`). `policy.toml` gets its schema: nodes, groups, per-target authority and requirements, and path rules, validated so a policy landing can never lock `policy` up. Targets fork with `origin target new` and land on each other with `origin land --from`. Their merge base comes from both histories (`ancestry.rs`): the best common ancestors, merged recursively when there are several, and any change under a path the bases disagree on conflicts. Conflicts are explicit state. A merge (`converge` now merges every head's files) writes both versions of a conflicted text file between markers, and a `core.conflict` lists each conflicted path with what was written there. A path stays unresolved until a revision changes it or `origin resolve` accepts it, every revision-producing operation carries unresolved paths forward, and the statements are published before their revision, so nothing finds the revision without them. Markers are only an editing aid; nothing scans files for them. --- AGENTS.md | 2 +- README.md | 23 + apps/origin-src/Cargo.toml | 4 + apps/origin-src/benches/annotations.rs | 20 +- apps/origin-src/benches/land.rs | 235 ++ apps/origin-src/src/ancestry.rs | 195 ++ apps/origin-src/src/annotations/mod.rs | 6 +- apps/origin-src/src/annotations/promotion.rs | 24 +- apps/origin-src/src/cli.rs | 62 +- apps/origin-src/src/conflict.rs | 416 ++++ apps/origin-src/src/lib.rs | 4 + apps/origin-src/src/policy.rs | 621 ++++++ apps/origin-src/src/queue.rs | 1966 +++++++++++++++++ apps/origin-src/src/store/mod.rs | 9 +- apps/origin-src/src/targets.rs | 31 + apps/origin-src/src/tree.rs | 369 +++- apps/origin-src/src/workspace/annotate.rs | 4 +- apps/origin-src/src/workspace/land.rs | 334 +++ apps/origin-src/src/workspace/mod.rs | 261 ++- apps/origin-src/tests/cli.rs | 378 +++- crates/origin-shared/src/object/acceptance.rs | 18 +- crates/origin-shared/src/object/mod.rs | 3 + crates/origin-shared/src/object/statement.rs | 29 +- docs/design.md | 6 +- 24 files changed, 4930 insertions(+), 90 deletions(-) create mode 100644 apps/origin-src/benches/land.rs create mode 100644 apps/origin-src/src/ancestry.rs create mode 100644 apps/origin-src/src/conflict.rs create mode 100644 apps/origin-src/src/policy.rs create mode 100644 apps/origin-src/src/queue.rs create mode 100644 apps/origin-src/src/workspace/land.rs diff --git a/AGENTS.md b/AGENTS.md index ada7970d..8be5a348 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,7 +27,7 @@ to a hot path (hashing, storage, snapshots, transfer) should come with a benchmark, and its numbers in the pull request. - `just bench`: the divan micro-benchmarks (encoding, store writes, - snapshots, input digests and promotion). + snapshots, input digests and promotion, landing). - `just bench-git`: times origin against git on a 20,000-file tree with hyperfine, and rewrites `docs/benchmarks.md`. diff --git a/README.md b/README.md index 862136bb..73cf8144 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ origin evolog # every revision of a change origin edit # switch the working copy to another change origin abandon --reason "Too slow" # give a change up, keeping it for others to see origin converge # end a change's divergence +origin resolve ... # accept conflicted files as they are origin index rebuild # rebuild the record index from the store origin annotate @ ci.test --pass --inputs src Cargo.toml # a statement about a revision @@ -31,8 +32,22 @@ origin annotate @ review --withdraw -m "Spoke too soon" # take your statement origin annotate --on flaky --neutral -m "Timeout on CI" # a statement about a statement origin statements [] # the statements about a revision, and those about them origin promote [] # carry results over from revisions with the same inputs + +origin land [] [--onto ] # land a change and the ones below it +origin land --from --onto # land a target's tip on another +origin land --status # the landings waiting for evidence or a conflict fix +origin targets # every target, its tip, authority and requirements +origin target new --from +origin new [description] --on # start a change on a target's tip ``` +Landing rebases a change onto the target's tip, file by file, checks the +statements `policy.toml` requires (promoting results whose inputs didn't +change), and appends an acceptance signed by your node, which must be the +target's authority. A landing that conflicts or lacks evidence waits: issue +the statements, or a new revision, and run `origin land` again. Policy is a +target too: edit it with `origin new --on policy`, then `origin land`. + Changes are named by a unique prefix of their id, or `@` for the working copy's. A revision is `@`, a change (for its head), or a unique prefix of the revision's id. Your name and node key live in `$ORIGIN_CONFIG_DIR`, or @@ -40,6 +55,14 @@ else `origin` in `$XDG_CONFIG_HOME` or `~/.config`: set `user = ""` in its `config.toml` if `$USER` isn't the name you want. See [the design](docs/design.md) for the model. +`converge` merges the files of a change's heads. A text file two heads +changed holds both versions between conflict markers (`<<<<<<<`, `=======`, +`>>>>>>>`), and a binary one keeps one side. A `core.conflict` statement +records which files conflict and what the merge wrote there, and the change +doesn't land while they're unchanged: edit them, or accept a file as it is +with `origin resolve `. `status` lists those left, and every new +revision, `describe` included, keeps them. + ## Development ```sh diff --git a/apps/origin-src/Cargo.toml b/apps/origin-src/Cargo.toml index 700c3eb6..fbab1ea8 100644 --- a/apps/origin-src/Cargo.toml +++ b/apps/origin-src/Cargo.toml @@ -37,6 +37,10 @@ tempfile.workspace = true name = "annotations" harness = false +[[bench]] +name = "land" +harness = false + [[bench]] name = "snapshot" harness = false diff --git a/apps/origin-src/benches/annotations.rs b/apps/origin-src/benches/annotations.rs index 16eb2ef1..a15505d8 100644 --- a/apps/origin-src/benches/annotations.rs +++ b/apps/origin-src/benches/annotations.rs @@ -180,12 +180,22 @@ fn promotions_cached(bencher: Bencher<'_, '_>) { let (_dir, store, target) = promotion_store(); let issuer: Principal = "node:queue".parse().unwrap(); let mut annotations = Annotations::open(&store).unwrap(); - assert_eq!(annotations.promotions(target, &issuer).unwrap().len(), 9); + assert_eq!( + annotations + .promotions(RecordId::Revision(target), &issuer) + .unwrap() + .len(), + 9 + ); let index = Index::open(&store).unwrap(); bencher .counter(ItemsCount::new(STATEMENTS)) .with_inputs(|| Annotations::new(&store, index.clone())) - .bench_local_refs(|annotations| annotations.promotions(target, &issuer).unwrap()); + .bench_local_refs(|annotations| { + annotations + .promotions(RecordId::Revision(target), &issuer) + .unwrap() + }); } /// The first search, which computes and remembers the digest of each of @@ -202,5 +212,9 @@ fn promotions_uncached(bencher: Bencher<'_, '_>) { fs::remove_file(&log).unwrap(); Annotations::open(&store).unwrap() }) - .bench_local_refs(|annotations| annotations.promotions(target, &issuer).unwrap()); + .bench_local_refs(|annotations| { + annotations + .promotions(RecordId::Revision(target), &issuer) + .unwrap() + }); } diff --git a/apps/origin-src/benches/land.rs b/apps/origin-src/benches/land.rs new file mode 100644 index 00000000..8a152d3e --- /dev/null +++ b/apps/origin-src/benches/land.rs @@ -0,0 +1,235 @@ +//! The landing queue: the latency of one landing, made on the target's tip +//! or rebased onto it, with and without a requirement, of a target landing +//! another after long histories, and the throughput of several proposers +//! landing different changes on one target at once. Run on a real disk: +//! syncs are most of the cost. + +use std::thread; + +use divan::Bencher; +use divan::counter::ItemsCount; +use origin_platform::Native; +use origin_shared::{ + AcceptanceId, Base, ChangeId, EncodedObject, EntryKind, EntryName, Kind, NodeKey, + PrincipalName, Record, RecordId, Revision, Statement, TargetName, Tree, TreeEntry, Verdict, +}; +use origin_src::annotations::Annotations; +use origin_src::queue::{Landing, Queue}; +use origin_src::store::Store; + +fn main() { + divan::main(); +} + +/// A repository whose `policy` and `main` the node `queue` orders. +struct Repo { + _dir: tempfile::TempDir, + store: Store, + key: NodeKey, +} + +impl Repo { + /// A repository where landing on `main` requires `require`, a TOML + /// array. + fn new(require: &str) -> Self { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let repo = Self { + _dir: dir, + store, + key: NodeKey::from_bytes(&[7; 32]), + }; + let policy = format!( + "[nodes]\nqueue = \"{}\"\n[groups]\nrunners = [\"node:ci\"]\n\ + [targets.policy]\nauthority = \"node:queue\"\n\ + [targets.feature]\nauthority = \"node:queue\"\n\ + [targets.main]\nauthority = \"node:queue\"\nrequire = {require}\n", + repo.key.public() + ); + repo.queue().init(&policy).unwrap(); + repo + } + + fn queue(&self) -> Queue<'_> { + Queue::new(&self.store, &self.key, PrincipalName::new("queue").unwrap()) + } + + fn tip(&self) -> AcceptanceId { + self.tip_of(&TargetName::MAIN) + } + + fn tip_of(&self, target: &TargetName) -> AcceptanceId { + self.store.head(target).unwrap().unwrap().acceptance + } + + /// Writes a change made on `base`, a tip of `main`, adding the file + /// `file-`. + fn change(&self, n: usize, base: AcceptanceId) -> ChangeId { + let mut tree: Tree = self + .store + .get(self.store.get(base).unwrap().body().state) + .unwrap(); + let content = EncodedObject::blob(format!("fn f{n}() {{}}\n").as_bytes()); + tree.insert( + EntryName::new(format!("file-{n}")).unwrap(), + TreeEntry { + kind: EntryKind::File, + id: content.id(), + }, + ); + let tree = tree.encode(); + let mut id = [0; 16]; + id[..8].copy_from_slice(&(n as u64).to_le_bytes()); + let change = ChangeId::from_bytes(id); + let revision = Revision { + change, + predecessors: Vec::new(), + base: Base::Acceptance(base), + deps: Vec::new(), + snapshot: tree.id().into(), + description: EncodedObject::blob(b"").id().into(), + author: "user:bench".parse().unwrap(), + } + .encode(); + let description = EncodedObject::blob(b""); + self.store.write_all(&[content, tree, description]).unwrap(); + self.store.write_all(&[revision]).unwrap(); + change + } + + fn land(&self, change: ChangeId) { + self.land_on(change, &TargetName::MAIN); + } + + fn land_on(&self, change: ChangeId, onto: &TargetName) { + let landings = self.queue().land_change(change, onto).unwrap(); + assert!( + matches!(landings[..], [Landing::Landed { .. }]), + "{landings:?}" + ); + } +} + +/// One landing of a change made on the tip: no rebase, no requirement. +#[divan::bench(sample_count = 50)] +fn land_on_tip(bencher: Bencher<'_, '_>) { + let repo = Repo::new("[]"); + let mut n = 0; + bencher + .with_inputs(|| { + n += 1; + repo.change(n, repo.tip()) + }) + .bench_local_values(|change| repo.land(change)); +} + +/// One landing of a change made on an older tip, which the queue rebases +/// into a new revision. +#[divan::bench(sample_count = 50)] +fn land_rebased(bencher: Bencher<'_, '_>) { + let repo = Repo::new("[]"); + let first = repo.tip(); + let mut n = 0; + bencher + .with_inputs(|| { + n += 1; + repo.change(n, first) + }) + .bench_local_values(|change| repo.land(change)); +} + +/// One landing of a change made on the tip, with the `ci.test` result +/// `main` requires. +#[divan::bench(sample_count = 50)] +fn land_with_requirement(bencher: Bencher<'_, '_>) { + let repo = Repo::new(r#"[{ kind = "ci.test", verdict = "pass", issuer = "group:runners" }]"#); + let mut n = 0; + bencher + .with_inputs(|| { + n += 1; + let change = repo.change(n, repo.tip()); + let annotations = Annotations::open(&repo.store).unwrap(); + let head = annotations.index().heads(change)[0]; + let result = Statement { + subject: RecordId::Revision(head), + kind: Kind::new("ci.test").unwrap(), + verdict: Some(Verdict::Pass), + issuer: "node:ci".parse().unwrap(), + inputs: None, + payload: None, + supersedes: None, + promoted_from: None, + }; + annotations.issue(&result, None).unwrap(); + change + }) + .bench_local_values(|change| repo.land(change)); +} + +/// How many changes each target lands before they merge. +const HISTORY: usize = 1_000; + +/// Landing the tip of `feature` on `main` once each landed `HISTORY` +/// changes since `feature` forked, after `feature` lands one more: the +/// merge base reads both histories whole. +#[divan::bench(sample_count = 20, sample_size = 1)] +fn land_target_after_long_histories(bencher: Bencher<'_, '_>) { + let repo = Repo::new("[]"); + let feature = TargetName::new("feature").unwrap(); + repo.queue() + .create_target(&feature, &TargetName::MAIN) + .unwrap(); + for n in 0..HISTORY { + repo.land(repo.change(n, repo.tip())); + let change = repo.change(HISTORY + n, repo.tip_of(&feature)); + repo.land_on(change, &feature); + } + let mut n = 2 * HISTORY; + bencher + .with_inputs(|| { + n += 1; + let change = repo.change(n, repo.tip_of(&feature)); + repo.land_on(change, &feature); + }) + .bench_local_values(|()| { + let landing = repo + .queue() + .land_target(&feature, &TargetName::MAIN) + .unwrap(); + assert!(matches!(landing, Landing::Landed { .. }), "{landing:?}"); + }); +} + +/// How many changes each proposer lands. +const EACH: usize = 8; + +/// `proposers` threads landing `EACH` changes each on `main` at once, all +/// made on its first tip, so all but the first are rebased. Threads lock +/// the queue as processes do: the lock belongs to the open file. +#[divan::bench(args = [1, 2, 4, 8], sample_count = 5, sample_size = 1)] +fn concurrent_proposers(bencher: Bencher<'_, '_>, proposers: usize) { + bencher + .counter(ItemsCount::new(proposers * EACH)) + .with_inputs(|| { + let repo = Repo::new("[]"); + let first = repo.tip(); + let changes: Vec<_> = (0..proposers * EACH) + .map(|n| repo.change(n, first)) + .collect(); + (repo, changes) + }) + .bench_local_values(|(repo, changes)| { + thread::scope(|scope| { + for changes in changes.chunks(EACH) { + let repo = &repo; + scope.spawn(move || { + for &change in changes { + repo.land(change); + } + }); + } + }); + (repo, changes) + }); +} diff --git a/apps/origin-src/src/ancestry.rs b/apps/origin-src/src/ancestry.rs new file mode 100644 index 00000000..8278e72f --- /dev/null +++ b/apps/origin-src/src/ancestry.rs @@ -0,0 +1,195 @@ +//! Merge bases in a history of immutable records with snapshots: the +//! acceptances of targets, each after the one before it and the +//! acceptances it landed, or the revisions of a change, each after its +//! predecessors. + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; + +use origin_platform::Platform; +use origin_shared::TreeId; + +use crate::store::{Store, StoreError}; + +/// A history, as far as it was walked: the parents and the snapshot of +/// each record in it. +#[derive(Debug)] +pub struct Ancestry { + records: HashMap, TreeId)>, +} + +impl Ancestry { + /// Walks everything `tips` descend from, asking `read` for the parents + /// and the snapshot of each record, once. + /// + /// # Errors + /// + /// Fails as `read` does. + pub fn walk( + tips: &[I], + mut read: impl FnMut(I) -> Result<(Vec, TreeId), E>, + ) -> Result { + let mut records = HashMap::new(); + let mut next = tips.to_vec(); + while let Some(id) = next.pop() { + if records.contains_key(&id) { + continue; + } + let record = read(id)?; + next.extend(&record.0); + records.insert(id, record); + } + Ok(Self { records }) + } + + /// The snapshot to merge `tips` from, `None` if they share no ancestor: + /// that of their best common ancestor. Several, as histories that + /// merged each other criss-cross have, merge into one first, as git's + /// recursive strategy does: each from their own merge base, with what + /// [`Store::merge_bases`] leaves where they conflict, so a merge from + /// the result conflicts there, and under it, unless both sides agree. Writes the trees + /// that makes, which merging reads. + /// + /// # Errors + /// + /// Fails if a tree cannot be read or written. + pub fn merge_base( + &self, + store: &Store

, + tips: &[I], + ) -> Result, StoreError> { + let groups: Vec<_> = tips.iter().map(std::slice::from_ref).collect(); + self.merge_base_of(store, &groups) + } + + /// The snapshot to merge `groups` from, each standing for all it holds. + fn merge_base_of( + &self, + store: &Store

, + groups: &[&[I]], + ) -> Result, StoreError> { + let bases = self.best_common(groups); + let Some((&first, others)) = bases.split_first() else { + return Ok(None); + }; + let mut merged = vec![first]; + let mut tree = self.snapshot(first); + for &other in others { + let base = self.merge_base_of(store, &[&merged, &[other]])?; + let merge = store.merge_bases(base, tree, self.snapshot(other))?; + store.write_all(&merge.trees)?; + tree = merge.tree; + merged.push(other); + } + Ok(Some(tree)) + } + + /// The best common ancestors of `groups`: the records that every group + /// has one that is or descends from, but that none of the others + /// descends from, sorted. Each record is visited a bounded number of + /// times. + fn best_common(&self, groups: &[&[I]]) -> Vec { + let Some((first, others)) = groups.split_first() else { + return Vec::new(); + }; + let mut common = self.reachable(first.iter().copied()); + for group in others { + let reachable = self.reachable(group.iter().copied()); + common.retain(|id| reachable.contains(id)); + } + let parents = common.iter().flat_map(|&id| self.parents(id)); + let below = self.reachable(parents.copied()); + let mut bases: Vec<_> = common.difference(&below).copied().collect(); + bases.sort_unstable(); + bases + } + + /// The records `from` are or descend from. + fn reachable(&self, from: impl IntoIterator) -> HashSet { + let mut reached = HashSet::new(); + let mut next: Vec<_> = from.into_iter().collect(); + while let Some(id) = next.pop() { + if reached.insert(id) { + next.extend(self.parents(id)); + } + } + reached + } + + /// Whether `id` is or descends from `ancestor`, as far as the walk + /// read. + #[must_use] + pub fn descends(&self, id: I, ancestor: I) -> bool { + self.reachable([id]).contains(&ancestor) + } + + fn parents(&self, id: I) -> &[I] { + self.records.get(&id).map_or(&[], |(parents, _)| parents) + } + + fn snapshot(&self, id: I) -> TreeId { + self.records[&id].1 + } +} + +#[cfg(test)] +mod tests { + use origin_shared::{Record, Tree}; + + use super::*; + + /// The history of `edges`, each a record and its parents. + fn ancestry(edges: &[(u8, &[u8])]) -> Ancestry { + let tips: Vec<_> = edges.iter().map(|&(id, _)| id).collect(); + let empty = Tree::default().encode().id().into(); + Ancestry::walk(&tips, |id| { + let parents = edges.iter().find(|&&(of, _)| of == id).map(|&(_, p)| p); + Ok::<_, ()>((parents.unwrap_or_default().to_vec(), empty)) + }) + .unwrap() + } + + fn bases(history: &Ancestry, tips: &[u8]) -> Vec { + let groups: Vec<_> = tips.iter().map(std::slice::from_ref).collect(); + history.best_common(&groups) + } + + #[test] + fn the_newest_shared_ancestor_wins_however_far() { + // 2 follows 1; 3 to 6 and 7 to 10 grow apart from 2, then each + // lands a side line forked from 1: 1 is nearer, but 2 is newer. + let history = ancestry(&[ + (2, &[1]), + (3, &[2]), + (4, &[3]), + (5, &[4]), + (6, &[5]), + (7, &[2]), + (8, &[7]), + (9, &[8]), + (10, &[9]), + (11, &[1]), + (12, &[6, 11]), + (13, &[1]), + (14, &[10, 13]), + ]); + assert_eq!(bases(&history, &[12, 14]), [2]); + assert_eq!(bases(&history, &[12, 6]), [6], "an ancestor of the other"); + } + + #[test] + fn criss_cross_merges_have_two_bases() { + // 2 and 3 fork from 1, then each merges the other. + let history = ancestry(&[(2, &[1]), (3, &[1]), (4, &[2, 3]), (5, &[3, 2])]); + assert_eq!(bases(&history, &[4, 5]), [2, 3]); + assert_eq!(bases(&history, &[4, 5, 3]), [3]); + // Merging those two needs their own base. + assert_eq!(history.best_common(&[&[2], &[3]]), [1]); + } + + #[test] + fn unrelated_histories_have_none() { + let history = ancestry(&[(2, &[1]), (4, &[3])]); + assert_eq!(bases(&history, &[2, 4]), []); + } +} diff --git a/apps/origin-src/src/annotations/mod.rs b/apps/origin-src/src/annotations/mod.rs index 60da8af6..a82ce0ee 100644 --- a/apps/origin-src/src/annotations/mod.rs +++ b/apps/origin-src/src/annotations/mod.rs @@ -300,7 +300,9 @@ pub enum AnnotateError { #[cfg(test)] mod tests { - use origin_shared::{Base, BlobId, Digest, Inputs, ObjectId, RepoPath, Revision, Verdict}; + use origin_shared::{ + Base, BlobId, Digest, Inputs, ObjectId, RepoPath, Revision, RevisionId, Verdict, + }; use super::*; use crate::tree::tests::build; @@ -382,7 +384,7 @@ mod tests { let promoter = principal("node:queue"); Annotations::open(store) .unwrap() - .promote(revision, &promoter) + .promote(RecordId::Revision(revision), &promoter) .unwrap() .into_iter() .map(|(_, statement)| { diff --git a/apps/origin-src/src/annotations/promotion.rs b/apps/origin-src/src/annotations/promotion.rs index abef909a..c2610504 100644 --- a/apps/origin-src/src/annotations/promotion.rs +++ b/apps/origin-src/src/annotations/promotion.rs @@ -17,7 +17,7 @@ use std::collections::HashSet; use origin_platform::Platform; -use origin_shared::{Principal, Record, RecordId, RevisionId, Statement, StatementId, TreeId}; +use origin_shared::{Principal, Record, RecordId, Statement, StatementId, TreeId}; use super::Annotations; use crate::store::{Store, StoreError}; @@ -140,13 +140,14 @@ impl Annotations<'_, P> { self.index.remember_digests(self.store, &digests.concat()) } - /// The statements promoting results to `revision`, as `issuer` would + /// The statements promoting results to `subject`, as `issuer` would /// issue them, without writing them, sorted by kind: for each /// [promotion group](Statement::promotion_group), the latest statement - /// about another record with the same input digest as the revision, + /// about another record with the same input digest as the subject, /// unless its issuer superseded it. Groups whose kind and issuer the - /// revision already has a result of, directly or promoted, are skipped: - /// a result that ran there wins. + /// subject already has a result of, directly or promoted, are skipped: + /// a result that ran there wins. A statement has no snapshot, so + /// nothing is promoted to one. /// /// # Errors /// @@ -154,11 +155,12 @@ impl Annotations<'_, P> { /// be remembered. pub fn promotions( &mut self, - revision: RevisionId, + subject: RecordId, issuer: &Principal, ) -> Result, StoreError> { - let subject = RecordId::Revision(revision); - let snapshot = self.store.get(revision)?.snapshot; + let Some(snapshot) = self.store.snapshot_of(subject)? else { + return Ok(Vec::new()); + }; self.compute_digests()?; // The results the revision has that hold, by kind and original @@ -203,7 +205,7 @@ impl Annotations<'_, P> { Ok(promotions) } - /// Issues the [`promotions`](Self::promotions) to `revision` as + /// Issues the [`promotions`](Self::promotions) to `subject` as /// `issuer`, returning them. /// /// # Errors @@ -212,10 +214,10 @@ impl Annotations<'_, P> { /// be written. pub fn promote( &mut self, - revision: RevisionId, + subject: RecordId, issuer: &Principal, ) -> Result, StoreError> { - let promotions = self.promotions(revision, issuer)?; + let promotions = self.promotions(subject, issuer)?; let encoded: Vec<_> = promotions.iter().map(Statement::encode).collect(); self.store.write_all(&encoded)?; Ok(encoded diff --git a/apps/origin-src/src/cli.rs b/apps/origin-src/src/cli.rs index d7cd1b76..b80df439 100644 --- a/apps/origin-src/src/cli.rs +++ b/apps/origin-src/src/cli.rs @@ -53,6 +53,9 @@ enum WorkspaceCommand { /// What the new change is for. #[arg(default_value = "")] description: String, + /// Start it on the tip of this target instead. + #[arg(long)] + on: Option, }, /// Describe what the current change is for. Describe { @@ -78,6 +81,12 @@ enum WorkspaceCommand { /// The change. change: String, }, + /// Accept conflicted files as they are, resolving their conflicts. + Resolve { + /// The files, from the root of the repository. + #[arg(required = true)] + paths: Vec, + }, /// Show the working copy's changes and the target they're on. Log { /// Show this target's history instead. @@ -119,6 +128,44 @@ enum WorkspaceCommand { #[arg(default_value = "@")] rev: String, }, + /// Land a change on a target, with the changes below it the target + /// doesn't have yet. The queue rebases it onto the target's tip, checks + /// the evidence `policy.toml` requires, and if it's there, appends a + /// signed acceptance. A landing that conflicts or lacks evidence waits. + Land(LandArgs), + /// Manage targets. + #[command(subcommand)] + Target(TargetCommand), + /// List the targets, with their tips, authorities and requirements. + Targets, +} + +#[derive(Debug, Args)] +struct LandArgs { + /// The change: the working copy's by default. + #[arg(conflicts_with_all = ["from", "status"])] + change: Option, + /// The target to land on: by default, the one the change is on. + #[arg(long)] + onto: Option, + /// Land the tip of this target instead of a change. + #[arg(long, requires = "onto")] + from: Option, + /// List the landings that wait, and why. + #[arg(long, conflicts_with_all = ["onto", "from"])] + status: bool, +} + +#[derive(Debug, Subcommand)] +enum TargetCommand { + /// Create a target at the tip of another. + New { + /// Its name, such as `release/v3`. + name: TargetName, + /// The target it starts from. + #[arg(long)] + from: TargetName, + }, } #[derive(Debug, Args)] @@ -331,8 +378,12 @@ impl WorkspaceCommand { Self::Status => workspace.status()?, Self::Diff => workspace.diff()?, Self::Show { change } => workspace.show(change.as_deref())?, - Self::New { description } => workspace.new_change(&description)?, + Self::New { description, on } => match on { + Some(target) => workspace.new_on(&target, &description)?, + None => workspace.new_change(&description)?, + }, Self::Describe { description } => workspace.describe(&description)?, + Self::Resolve { paths } => workspace.resolve(&paths)?, Self::Edit { change } => workspace.edit(&change)?, Self::Abandon { change, reason } => workspace.abandon(change.as_deref(), &reason)?, Self::Converge { change } => workspace.converge(&change)?, @@ -351,6 +402,15 @@ impl WorkspaceCommand { } Self::Statements { rev } => workspace.statements(&rev)?, Self::Promote { rev } => workspace.promote(&rev)?, + Self::Land(args) => match (args.from, args.onto) { + _ if args.status => workspace.land_status()?, + (Some(from), Some(onto)) => workspace.land_target(&from, &onto)?, + (_, onto) => workspace.land(args.change.as_deref(), onto)?, + }, + Self::Target(TargetCommand::New { name, from }) => { + workspace.target_new(&name, &from)?; + } + Self::Targets => workspace.targets()?, } Ok(()) } diff --git a/apps/origin-src/src/conflict.rs b/apps/origin-src/src/conflict.rs new file mode 100644 index 00000000..4367cad4 --- /dev/null +++ b/apps/origin-src/src/conflict.rs @@ -0,0 +1,416 @@ +//! Conflicts: what a merge leaves where both sides changed a file. +//! +//! A merge into a working copy, such as `origin converge`, writes both +//! versions of each such text file, between markers naming the sides: +//! +//! ```text +//! <<<<<<< 3f1c9a2e07b4 +//! our version +//! ======= +//! their version +//! >>>>>>> 8d02e5b1c4f9 +//! ``` +//! +//! A binary file, or a path a directory is on either side of, keeps one +//! side instead. The merge then issues a `core.conflict` statement about +//! the new revision, as its author, a user, listing each conflicted path +//! and the entry it wrote there. That list is the conflict, not the +//! markers, which only help to edit the file: a path stays unresolved while +//! a revision has the entry written there. Changing the file resolves it, +//! and so does `origin resolve`, which accepts it as it is, such as the +//! side a binary file kept. +//! +//! Every revision made from one with unresolved paths, by a snapshot, +//! `describe`, a new change on top, or a merge, carries those it still has +//! the entries of over, in a `core.conflict` of its own, published before +//! the revision itself, so it's never found without it. The landing queue +//! refuses a revision whose author's latest `core.conflict` lists a path it +//! still has the entry of. +//! +//! The landing queue also issues `core.conflict`, as a node, when a revision +//! doesn't rebase onto its target, even one the queue rebased itself. That +//! one only blocks the landing on that target, which merges again whenever +//! it's retried. + +use origin_platform::Platform; +use origin_shared::{ + EncodedObject, Kind, ObjectId, Principal, RecordId, RepoPath, Revision, Statement, StatementId, + TreeId, +}; + +use crate::annotations::Claim; +use crate::store::{Store, StoreError}; +use crate::tree::Snapshots; + +/// A path a merge left conflicted, and the entry it wrote there. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Conflicted { + /// The path. + pub path: RepoPath, + /// The id of the entry written there: both versions between markers, + /// or the side kept. `None` if the merge left nothing there. + pub written: Option, +} + +/// Conflicted paths, sorted, each at most once: those a merge left, or a +/// revision's unresolved ones. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Conflicts(Vec); + +impl Conflicts { + /// The line starting our side, then its label. + const OURS: &str = "<<<<<<< "; + /// The line between the sides. + const BETWEEN: &str = "======="; + /// The line ending their side, then its label. + const THEIRS: &str = ">>>>>>> "; + + /// The conflicts of `conflicted`, the latest of each path counting. + #[must_use] + pub fn new(mut conflicted: Vec) -> Self { + // Stable, so the latest of each path comes last, and dedup keeps the + // first of equal ones: reverse around it. + conflicted.reverse(); + conflicted.sort_by(|a, b| a.path.cmp(&b.path)); + conflicted.dedup_by(|a, b| a.path == b.path); + Self(conflicted) + } + + /// Adds `conflicted`, which must come after every path listed. + pub(crate) fn push(&mut self, conflicted: Conflicted) { + debug_assert!(self.0.last().is_none_or(|last| last.path < conflicted.path)); + self.0.push(conflicted); + } + + /// Whether there are none. + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// The conflicted paths, sorted. + #[must_use] + pub fn paths(&self) -> Vec { + self.0 + .iter() + .map(|conflicted| conflicted.path.clone()) + .collect() + } + + /// These conflicts, then `newer` ones, which replace those of the same + /// paths: what a merge of a revision with these conflicts leaves. + #[must_use] + pub fn with(mut self, newer: Self) -> Self { + self.0.extend(newer.0); + Self::new(self.0) + } + + /// Resolves the conflicts of `paths`, returning those it had. + pub fn resolve(&mut self, paths: &[RepoPath]) -> Vec { + let mut resolved = Vec::new(); + self.0.retain(|conflicted| { + let listed = paths.contains(&conflicted.path); + if listed { + resolved.push(conflicted.path.clone()); + } + !listed + }); + resolved + } + + /// The content of a file both sides changed: `ours`, then `theirs`, + /// between markers naming them with `labels`. A side that removed the + /// file is empty. `None` if either side is binary, holding a NUL byte: + /// markers would only corrupt it. + #[must_use] + pub fn mark(ours: &[u8], theirs: &[u8], labels: [&str; 2]) -> Option> { + if ours.contains(&0) || theirs.contains(&0) { + return None; + } + let mut marked = Vec::with_capacity(ours.len() + theirs.len() + 64); + let side = |marked: &mut Vec, content: &[u8]| { + marked.extend_from_slice(content); + if !content.is_empty() && !content.ends_with(b"\n") { + marked.push(b'\n'); + } + }; + marked.extend_from_slice(format!("{}{}\n", Self::OURS, labels[0]).as_bytes()); + side(&mut marked, ours); + marked.extend_from_slice(format!("{}\n", Self::BETWEEN).as_bytes()); + side(&mut marked, theirs); + marked.extend_from_slice(format!("{}{}\n", Self::THEIRS, labels[1]).as_bytes()); + Some(marked) + } + + /// The `core.conflict` statement about `subject`, from `issuer`, + /// listing these conflicts in its payload, and the payload: each one + /// as the id written, or `-` for none, a space and the path, ended by a + /// NUL byte, which no path holds. + #[must_use] + pub fn statement( + &self, + subject: RecordId, + issuer: Principal, + supersedes: Option, + ) -> (EncodedObject, Statement) { + let mut list = String::new(); + for conflicted in &self.0 { + match conflicted.written { + Some(id) => list.push_str(&id.to_string()), + None => list.push('-'), + } + list.push(' '); + list.push_str(conflicted.path.as_str()); + list.push('\0'); + } + let payload = EncodedObject::blob(list.as_bytes()); + let statement = Statement { + subject, + kind: Kind::CONFLICT, + verdict: None, + issuer, + inputs: None, + payload: Some(payload.id().into()), + supersedes, + promoted_from: None, + }; + (payload, statement) + } + + /// The conflicts a `core.conflict` statement lists. Its payload is the + /// list, which a repository always keeps, so one it lacks fails rather + /// than reading as no conflicts. + /// + /// # Errors + /// + /// Fails if the payload is missing or cannot be read. + pub fn listed( + store: &Store

, + statement: &Statement, + ) -> Result { + let Some(payload) = statement.payload else { + return Ok(Self::default()); + }; + let content = store + .try_blob(payload.into())? + .ok_or(StoreError::NotFound(payload.into()))?; + let conflicted = String::from_utf8_lossy(&content) + .split_terminator('\0') + .filter_map(|line| { + let (written, path) = line.split_once(' ')?; + let written = match written { + "-" => None, + id => Some(id.parse().ok()?), + }; + let path = RepoPath::new(path).ok()?; + Some(Conflicted { path, written }) + }) + .collect(); + Ok(Self::new(conflicted)) + } + + /// The unresolved conflicts of `revision`, from `claims`, those about it + /// that hold: of the latest `core.conflict` from its author, if that's + /// a user, the paths where it still has the entry written. A node's + /// says only that a landing queue could not rebase it. + /// + /// # Errors + /// + /// Fails if the statement's payload is missing, or it or a tree on the + /// way to a path cannot be read. + pub fn of( + store: &Store

, + claims: &[Claim], + revision: &Revision, + ) -> Result { + if matches!(revision.author, Principal::Node(_)) { + return Ok(Self::default()); + } + let Some(claim) = claims.iter().rev().find(|claim| { + claim.statement().kind == Kind::CONFLICT && claim.statement().issuer == revision.author + }) else { + return Ok(Self::default()); + }; + Self::listed(store, claim.statement())?.remaining(store, revision.snapshot) + } + + /// Those of these conflicts `snapshot` still has: where it has the + /// entry written. + /// + /// # Errors + /// + /// Fails if a tree on the way to a path cannot be read. + pub fn remaining( + mut self, + store: &Store

, + snapshot: TreeId, + ) -> Result { + let mut snapshots = Snapshots::new(store); + let mut remaining = Vec::with_capacity(self.0.len()); + for conflicted in self.0.drain(..) { + let entry = snapshots.entry_at(snapshot, &conflicted.path)?; + if entry.map(|entry| entry.id) == conflicted.written { + remaining.push(conflicted); + } + } + Ok(Self(remaining)) + } +} + +#[cfg(test)] +mod tests { + use origin_platform::Native; + use origin_shared::{EntryKind, EntryName, Record, Tree, TreeEntry}; + + use super::*; + + fn path(path: &str) -> RepoPath { + RepoPath::new(path).unwrap() + } + + #[test] + fn a_path_resolves_once_its_entry_changes() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let marked = EncodedObject::blob(b"<<<<<<< a\nours\n=======\ntheirs\n>>>>>>> b\n"); + let resolved = EncodedObject::blob(b"Heading\n=======\nresolved\n"); + let snapshot = |file: &EncodedObject| { + let mut tree = Tree::default(); + tree.insert( + EntryName::new("file").unwrap(), + TreeEntry { + kind: EntryKind::File, + id: file.id(), + }, + ); + let tree = tree.encode(); + store.write_all(&[file.clone(), tree.clone()]).unwrap(); + TreeId::from(tree.id()) + }; + let conflicts = Conflicts::new(vec![ + Conflicted { + path: path("file"), + written: Some(marked.id()), + }, + Conflicted { + path: path("removed"), + written: None, + }, + ]); + let (payload, statement) = conflicts.statement( + RecordId::Snapshot(snapshot(&marked)), + "user:ada".parse().unwrap(), + None, + ); + // Without its payload, the statement lists nothing it can trust. + assert!(matches!( + Conflicts::listed(&store, &statement), + Err(StoreError::NotFound(id)) if id == payload.id() + )); + store.write_all(&[payload]).unwrap(); + assert_eq!(Conflicts::listed(&store, &statement).unwrap(), conflicts); + + let unchanged = conflicts.clone().remaining(&store, snapshot(&marked)); + assert_eq!(unchanged.unwrap(), conflicts); + // Even to content that looks like a marker. + let remaining = conflicts.remaining(&store, snapshot(&resolved)).unwrap(); + assert_eq!(remaining.paths(), [path("removed")]); + } + + #[test] + fn only_a_user_author_leaves_a_revision_unresolved() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let tree = Tree::default().encode(); + store.write_all(std::slice::from_ref(&tree)).unwrap(); + let revision = |author: &str| Revision { + change: origin_shared::ChangeId::from_bytes([1; 16]), + predecessors: Vec::new(), + base: origin_shared::Base::Acceptance(EncodedObject::blob(b"base").id().into()), + deps: Vec::new(), + snapshot: tree.id().into(), + description: EncodedObject::blob(b"").id().into(), + author: author.parse().unwrap(), + }; + let subject = RecordId::Revision(revision("user:ada").encode().id().into()); + let conflicts = Conflicts::new(vec![Conflicted { + path: path("file"), + written: None, + }]); + let claims = |issuer: &str| { + let (payload, statement) = conflicts.statement(subject, issuer.parse().unwrap(), None); + store.write_all(&[payload]).unwrap(); + let id = statement.encode().id().into(); + [store.claim(id, statement).unwrap().unwrap()] + }; + let ada = revision("user:ada"); + assert_eq!( + Conflicts::of(&store, &claims("user:ada"), &ada).unwrap(), + conflicts + ); + assert!( + Conflicts::of(&store, &claims("user:eve"), &ada) + .unwrap() + .is_empty() + ); + // A queue that rebased a revision, then could not rebase it elsewhere. + let queue = revision("node:queue"); + assert!( + Conflicts::of(&store, &claims("node:queue"), &queue) + .unwrap() + .is_empty() + ); + } + + #[test] + fn newer_conflicts_replace_older_ones() { + let older = Conflicts::new(vec![ + Conflicted { + path: path("a"), + written: None, + }, + Conflicted { + path: path("b"), + written: None, + }, + ]); + let id = EncodedObject::blob(b"newer").id(); + let newer = Conflicts::new(vec![Conflicted { + path: path("b"), + written: Some(id), + }]); + let mut both = older.with(newer); + assert_eq!( + both, + Conflicts::new(vec![ + Conflicted { + path: path("a"), + written: None, + }, + Conflicted { + path: path("b"), + written: Some(id), + }, + ]) + ); + assert_eq!(both.resolve(&[path("a"), path("c")]), [path("a")]); + assert_eq!(both.paths(), [path("b")]); + } + + #[test] + fn marks_both_sides() { + let marked = Conflicts::mark(b"ours\n", b"theirs", ["a", "b"]).unwrap(); + assert_eq!( + String::from_utf8(marked).unwrap(), + "<<<<<<< a\nours\n=======\ntheirs\n>>>>>>> b\n" + ); + assert_eq!( + Conflicts::mark(b"", b"kept\n", ["a", "b"]).unwrap(), + b"<<<<<<< a\n=======\nkept\n>>>>>>> b\n", + "a removed side is empty" + ); + assert_eq!(Conflicts::mark(b"\0binary", b"text", ["a", "b"]), None); + } +} diff --git a/apps/origin-src/src/lib.rs b/apps/origin-src/src/lib.rs index 94161e76..f00f28fc 100644 --- a/apps/origin-src/src/lib.rs +++ b/apps/origin-src/src/lib.rs @@ -1,13 +1,17 @@ //! The origin command line and server. Both share the repository logic in //! this library; `main.rs` only parses arguments and dispatches. +pub mod ancestry; pub mod annotations; pub mod cli; pub mod config; +pub mod conflict; #[cfg(feature = "fixture")] pub mod fixture; pub mod index; pub mod output; +pub mod policy; +pub mod queue; pub mod store; pub mod targets; pub mod tree; diff --git a/apps/origin-src/src/policy.rs b/apps/origin-src/src/policy.rs new file mode 100644 index 00000000..61a67b6f --- /dev/null +++ b/apps/origin-src/src/policy.rs @@ -0,0 +1,621 @@ +//! `policy.toml`, in the snapshot of the `policy` target: the nodes and +//! groups of a repository, and who orders each target and what landing on +//! it requires. The policy in force is the one at the target's head. +//! +//! ```toml +//! [nodes] # each node's ed25519 public key, in hex +//! hosted = "3b6a27bc..." +//! +//! [groups] # users and nodes +//! maintainers = ["user:alice"] +//! runners = ["node:ci-1", "node:ci-2"] +//! +//! [targets.main] +//! authority = "node:hosted" # the one node that orders the target +//! require = [ +//! { kind = "ci.test", verdict = "pass", issuer = "group:runners" }, +//! { kind = "review", verdict = "pass", issuer = "group:maintainers" }, +//! ] +//! +//! [[paths]] +//! path = "secrets" +//! grants = { all = "none", "group:maintainers" = "write" } +//! ``` +//! +//! A target without a table of its own has the authority of `policy`, and +//! requires nothing. Every name a policy refers to must be defined in it, +//! and unknown keys are errors, so a typo never drops a requirement. Path +//! rules are parsed and checked, but not enforced yet. + +use std::collections::BTreeMap; +use std::fmt; +use std::str::FromStr; + +use origin_platform::Platform; +use origin_shared::{ + AcceptanceBody, AcceptanceId, EncodedObject, EntryKind, EntryName, Kind, NodePublicKey, + Principal, PrincipalName, Record, RepoPath, TargetName, Tree, TreeEntry, TreeId, Verdict, +}; +use serde::Deserialize; + +use crate::store::{Store, StoreError}; + +/// The policy of a repository. +#[derive(Debug, Default, Deserialize)] +#[serde(try_from = "PolicyFile")] +pub struct Policy { + nodes: BTreeMap, + groups: BTreeMap>, + targets: BTreeMap, + paths: Vec, +} + +/// Who orders a target, and what landing on it requires. +#[derive(Clone, Debug, PartialEq, Eq)] +struct TargetRules { + authority: PrincipalName, + require: Vec, +} + +/// A statement that landing needs: of `kind`, with `verdict`, from one of +/// `issuer`, about what lands. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Requirement { + /// The statement's kind. + pub kind: Kind, + /// Its verdict. + pub verdict: Verdict, + /// Who may issue it. + pub issuer: Principals, +} + +/// One user or node, or the members of a group: `user:alice`, +/// `node:ci-1` or `group:runners`. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Principals { + /// A user or a node. + One(Principal), + /// The members of a group of the policy. + Group(PrincipalName), +} + +/// A `[[paths]]` rule: what each grantee may do under a path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PathRule { + /// The file or directory the rule covers, with everything under it. + pub path: RepoPath, + /// What each grantee may do there. + pub grants: BTreeMap, +} + +/// Who a path rule grants access to. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Grantee { + /// `all`: everyone. + All, + /// A user, a node or a group's members. + Principals(Principals), +} + +/// What a path rule grants. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Access { + /// Nothing, not even the content. + None, + /// Reading the content. + Read, + /// Proposing revisions that change it. + Write, +} + +impl Policy { + /// The file holding the policy, at the root of the `policy` target. + pub const FILE: &str = "policy.toml"; + + /// The acceptance of `policy` whose `policy.toml` governs the acceptance + /// `body`, `id`: who must have signed it, and what it had to meet. An + /// acceptance of `policy` is governed by the one before it, which it + /// must name, so a policy never authorizes its own landing. Only the + /// first, which has none before it, is governed by its own. Any other + /// acceptance is governed by the policy it names, which the caller + /// checks is on `policy`'s chain. The landing queue checks what it + /// signs with it, and so must whatever verifies a chain. + /// + /// # Errors + /// + /// Fails if the acceptance names another policy than that rule's, or + /// none. + pub fn governing(id: AcceptanceId, body: &AcceptanceBody) -> Result { + if body.target != TargetName::POLICY.id() { + return body.policy.ok_or(PolicyError::Unnamed(body.seq)); + } + match (body.prev, body.policy) { + (None, None) => Ok(id), + (Some(prev), Some(policy)) if policy == prev => Ok(prev), + _ => Err(PolicyError::NotPredecessor(body.seq)), + } + } + + /// The policy in force, at the head of the `policy` target, and the + /// acceptance holding it. + /// + /// # Errors + /// + /// Fails if the repository has no policy, its records cannot be read, + /// or it is invalid. + pub fn in_force(store: &Store

) -> Result<(AcceptanceId, Self), PolicyError> { + let head = store + .head(&TargetName::POLICY)? + .ok_or(PolicyError::Missing)?; + let state = store.get(head.acceptance)?.body().state; + let policy = Self::of_snapshot(store, state)?.ok_or(PolicyError::Missing)?; + Ok((head.acceptance, policy)) + } + + /// The policy in force, or the default if the repository has none. + /// + /// # Errors + /// + /// Fails like [`Policy::in_force`], but for a missing policy. + pub fn load(store: &Store

) -> Result { + match Self::in_force(store) { + Ok((_, policy)) => Ok(policy), + Err(PolicyError::Missing) => Ok(Self::default()), + Err(error) => Err(error), + } + } + + /// The policy in `snapshot`, or `None` if it has no `policy.toml`. + /// + /// # Errors + /// + /// Fails if the snapshot cannot be read, or its `policy.toml` is + /// invalid. + pub fn of_snapshot( + store: &Store

, + snapshot: TreeId, + ) -> Result, PolicyError> { + let Some(&entry) = store.get(snapshot)?.get(Self::FILE) else { + return Ok(None); + }; + let content = store + .try_blob(entry.id)? + .ok_or(StoreError::NotFound(entry.id))?; + let content = String::from_utf8(content).map_err(|_| PolicyError::NotText(snapshot))?; + toml::from_str(&content) + .map(Some) + .map_err(|source| PolicyError::Invalid { + snapshot, + source: Box::new(source), + }) + } + + /// A snapshot holding only `policy` as its `policy.toml`: the file and + /// the tree. + pub(crate) fn snapshot(policy: &str) -> (EncodedObject, EncodedObject) { + let file = EncodedObject::blob(policy.as_bytes()); + let entry = TreeEntry { + kind: EntryKind::File, + id: file.id(), + }; + let name = EntryName::new(Self::FILE).expect("`policy.toml` is a valid name"); + let tree: Tree = [(name, entry)].into_iter().collect(); + (file, tree.encode()) + } + + /// The node that orders `target`: the one its table names, or else the + /// authority of `policy`. `None` if neither has a table. + #[must_use] + pub fn authority(&self, target: &TargetName) -> Option<&PrincipalName> { + self.targets + .get(target) + .or_else(|| self.targets.get(&TargetName::POLICY)) + .map(|rules| &rules.authority) + } + + /// What landing on `target` requires: nothing, if it has no table. + #[must_use] + pub fn requirements(&self, target: &TargetName) -> &[Requirement] { + self.targets + .get(target) + .map_or(&[], |rules| rules.require.as_slice()) + } + + /// The public key of `node`, if the policy lists it. + #[must_use] + pub fn node_key(&self, node: &PrincipalName) -> Option<&NodePublicKey> { + self.nodes.get(node) + } + + /// Whether `principal` is one of `principals`. + #[must_use] + pub fn admits(&self, principals: &Principals, principal: &Principal) -> bool { + match principals { + Principals::One(one) => one == principal, + Principals::Group(group) => self + .groups + .get(group) + .is_some_and(|members| members.contains(principal)), + } + } + + /// The `[[paths]]` rules, in order. + #[must_use] + pub fn paths(&self) -> &[PathRule] { + &self.paths + } +} + +/// `policy.toml` as written, which [`Policy::try_from`] parses and checks. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PolicyFile { + #[serde(default)] + nodes: BTreeMap, + #[serde(default)] + groups: BTreeMap>, + #[serde(default)] + targets: BTreeMap, + #[serde(default)] + paths: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TargetFile { + authority: String, + #[serde(default)] + require: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RequirementFile { + kind: String, + verdict: String, + issuer: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PathRuleFile { + path: String, + grants: BTreeMap, +} + +impl PolicyFile { + /// Parses `value`, which is a `what`. + fn parse(what: &'static str, value: String) -> Result + where + T: FromStr, + T::Err: fmt::Display, + { + value.parse().map_err(|error: T::Err| RuleError::Invalid { + what, + reason: error.to_string(), + value, + }) + } +} + +impl TryFrom for Policy { + type Error = RuleError; + + fn try_from(file: PolicyFile) -> Result { + let nodes = file + .nodes + .into_iter() + .map(|(name, key)| { + Ok(( + PolicyFile::parse("node name", name)?, + PolicyFile::parse("node key", key)?, + )) + }) + .collect::>()?; + let groups = file + .groups + .into_iter() + .map(|(name, members)| { + let members = members + .into_iter() + .map(|member| PolicyFile::parse("group member", member)) + .collect::>()?; + Ok((PolicyFile::parse("group name", name)?, members)) + }) + .collect::>()?; + let mut policy = Self { + nodes, + groups, + targets: BTreeMap::new(), + paths: Vec::new(), + }; + for (name, target) in file.targets { + let name: TargetName = PolicyFile::parse("target name", name)?; + let authority = match PolicyFile::parse("authority", target.authority)? { + Principal::Node(node) if policy.nodes.contains_key(&node) => node, + Principal::Node(node) => return Err(RuleError::UnknownNode { target: name, node }), + user @ Principal::User(_) => { + return Err(RuleError::NotANode { + target: name, + authority: user, + }); + } + }; + let require = target + .require + .into_iter() + .map(|requirement| { + Ok(Requirement { + kind: PolicyFile::parse("kind", requirement.kind)?, + verdict: PolicyFile::parse("verdict", requirement.verdict)?, + issuer: policy.check(PolicyFile::parse("issuer", requirement.issuer)?)?, + }) + }) + .collect::>()?; + policy + .targets + .insert(name, TargetRules { authority, require }); + } + for rule in file.paths { + let grants = rule + .grants + .into_iter() + .map(|(grantee, access)| { + let grantee = match PolicyFile::parse("grantee", grantee)? { + Grantee::Principals(principals) => { + Grantee::Principals(policy.check(principals)?) + } + Grantee::All => Grantee::All, + }; + Ok((grantee, PolicyFile::parse("access", access)?)) + }) + .collect::>()?; + policy.paths.push(PathRule { + path: PolicyFile::parse("path", rule.path)?, + grants, + }); + } + Ok(policy) + } +} + +impl Policy { + /// Checks that the group `principals` names, if any, is defined. + fn check(&self, principals: Principals) -> Result { + match &principals { + Principals::Group(group) if !self.groups.contains_key(group) => { + Err(RuleError::UnknownGroup(group.clone())) + } + _ => Ok(principals), + } + } +} + +impl FromStr for Principals { + type Err = String; + + fn from_str(input: &str) -> Result { + match input.strip_prefix("group:") { + Some(group) => PrincipalName::new(group) + .map(Self::Group) + .map_err(|error| error.to_string()), + None => input.parse().map(Self::One).map_err(|_| { + format!("`{input}` is not `user:`, `node:` or `group:`") + }), + } + } +} + +impl fmt::Display for Principals { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::One(principal) => principal.fmt(f), + Self::Group(group) => write!(f, "group:{group}"), + } + } +} + +impl FromStr for Grantee { + type Err = String; + + fn from_str(input: &str) -> Result { + if input == "all" { + return Ok(Self::All); + } + input + .parse() + .map(Self::Principals) + .map_err(|error| format!("{error}, or `all`")) + } +} + +impl FromStr for Access { + type Err = String; + + fn from_str(input: &str) -> Result { + match input { + "none" => Ok(Self::None), + "read" => Ok(Self::Read), + "write" => Ok(Self::Write), + _ => Err(format!("`{input}` is not `none`, `read` or `write`")), + } + } +} + +impl fmt::Display for Requirement { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} {} from {}", self.kind, self.verdict, self.issuer) + } +} + +/// An invalid rule in `policy.toml`. +#[derive(Debug, PartialEq, Eq, thiserror::Error)] +pub enum RuleError { + /// A value doesn't parse. + #[error("{what}: {reason}")] + Invalid { + /// What the value is. + what: &'static str, + /// The value. + value: String, + /// Why it's invalid. + reason: String, + }, + /// A target's authority isn't a node. + #[error("the authority of `{target}` is {authority}, but only a node can be one")] + NotANode { + /// The target. + target: TargetName, + /// Its authority. + authority: Principal, + }, + /// A target's authority has no key in `[nodes]`. + #[error("the authority of `{target}`, node:{node}, has no key in [nodes]")] + UnknownNode { + /// The target. + target: TargetName, + /// Its authority. + node: PrincipalName, + }, + /// A rule names a group `[groups]` doesn't define. + #[error("group:{0} is not defined in [groups]")] + UnknownGroup(PrincipalName), +} + +/// A failure to read the policy. +#[derive(Debug, thiserror::Error)] +pub enum PolicyError { + /// The repository has no `policy` target, or it holds no `policy.toml`. + #[error("the repository has no policy: `policy` holds no policy.toml")] + Missing, + /// The policy's records cannot be read. + #[error(transparent)] + Store(#[from] StoreError), + /// `policy.toml` isn't UTF-8. + #[error("the policy.toml of snapshot {} is not text", .0.short())] + NotText(TreeId), + /// `policy.toml` isn't a valid policy. + #[error("invalid policy.toml in snapshot {}", snapshot.short())] + Invalid { + /// The snapshot holding it. + snapshot: TreeId, + /// What is wrong, boxed: it's large, and rare. + source: Box, + }, + /// An acceptance of `policy`, at this sequence number, names another + /// policy than the acceptance before it. + #[error("acceptance #{0} of `policy` names another policy than the one before it")] + NotPredecessor(u64), + /// An acceptance of another target, at this sequence number, names no + /// policy. + #[error("acceptance #{0} names no policy")] + Unnamed(u64), +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A policy with every table, whose node `hosted` has the key of the + /// secret `[42; 32]`. + fn full_policy() -> String { + let key = origin_shared::NodeKey::from_bytes(&[42; 32]).public(); + format!( + "[nodes]\nhosted = \"{key}\"\n\ + [groups]\nrunners = [\"node:ci-1\"]\nmaintainers = [\"user:alice\"]\n\ + [targets.policy]\nauthority = \"node:hosted\"\n\ + [targets.main]\nauthority = \"node:hosted\"\nrequire = [\n\ + {{ kind = \"ci.test\", verdict = \"pass\", issuer = \"group:runners\" }},\n\ + {{ kind = \"review\", verdict = \"pass\", issuer = \"user:bob\" }},\n]\n\ + [[paths]]\npath = \"secrets\"\n\ + grants = {{ all = \"none\", \"group:maintainers\" = \"write\" }}\n" + ) + } + + fn principal(principal: &str) -> Principal { + principal.parse().unwrap() + } + + #[test] + fn parses_every_table() { + let policy: Policy = toml::from_str(&full_policy()).unwrap(); + let hosted = PrincipalName::new("hosted").unwrap(); + assert_eq!( + policy.node_key(&hosted), + Some(&origin_shared::NodeKey::from_bytes(&[42; 32]).public()) + ); + let main = TargetName::MAIN; + assert_eq!(policy.authority(&main), Some(&hosted)); + let required: Vec<_> = policy + .requirements(&main) + .iter() + .map(ToString::to_string) + .collect(); + assert_eq!( + required, + [ + "ci.test pass from group:runners", + "review pass from user:bob" + ] + ); + let runners = &policy.requirements(&main)[0].issuer; + assert!(policy.admits(runners, &principal("node:ci-1"))); + assert!( + !policy.admits(runners, &principal("user:ci-1")), + "a user isn't the node of the same name" + ); + + // A target without a table: the policy's authority, no requirements. + let other = TargetName::new("overhaul").unwrap(); + assert_eq!(policy.authority(&other), Some(&hosted)); + assert_eq!(policy.requirements(&other), []); + + let [rule] = policy.paths() else { + panic!("one path rule") + }; + assert_eq!(rule.path, RepoPath::new("secrets").unwrap()); + assert_eq!(rule.grants.get(&Grantee::All), Some(&Access::None)); + } + + #[test] + fn rejects_rules_naming_what_it_does_not_define() { + let invalid = |from: &str, to: &str| { + let toml = full_policy().replace(from, to); + assert_ne!(toml, full_policy(), "{from}"); + toml::from_str::(&toml).unwrap_err().to_string() + }; + for (from, to, expected) in [ + ( + "node:hosted", + "node:nobody", + "node:nobody, has no key in [nodes]", + ), + ( + "authority = \"node:hosted\"\nrequire", + "authority = \"user:hosted\"\nrequire", + "only a node", + ), + ( + "group:runners", + "group:nobody", + "group:nobody is not defined", + ), + ("group:maintainers", "group:x", "group:x is not defined"), + ("issuer", "issuers", "unknown field `issuers`"), + ( + "verdict = \"pass\"", + "verdict = \"ok\"", + "verdict: invalid verdict \"ok\"", + ), + ("\"write\"", "\"admin\"", "access: `admin` is not `none`"), + ( + "node:ci-1", + "ci-1", + "group member: invalid principal \"ci-1\"", + ), + ] { + let error = invalid(from, to); + assert!(error.contains(expected), "{error}"); + } + } +} diff --git a/apps/origin-src/src/queue.rs b/apps/origin-src/src/queue.rs new file mode 100644 index 00000000..8151009d --- /dev/null +++ b/apps/origin-src/src/queue.rs @@ -0,0 +1,1966 @@ +//! The landing queue: how a target's authority lands changes and other +//! targets on it, one acceptance at a time. +//! +//! Landing a change lands its stack: the head of each change below it that +//! isn't on the target yet, from the bottom, with an acceptance each. A +//! change that diverged, that its creator abandoned, or whose head has +//! unresolved conflicts (see [`conflict`](crate::conflict)) is refused. A +//! stack that waits is kept whole, so retrying it lands the rest once the +//! change it waits on can. For each revision: +//! +//! 1. **Rebase.** The changes it made to its base are merged onto the +//! target's tip, file by file ([`Store::merge`]). A revision made on the +//! tip lands as it is; any other is rebased into a new revision of its +//! change, on the tip, by this node. A landing that would leave the +//! target as it is lands nothing: the target already has the change. +//! 2. **Conflicts.** If both sides changed a path, a `core.conflict` +//! statement about the revision lists the paths, and the change waits, +//! blocked, until a new revision of it merges. +//! 3. **Policy.** The policy in force now must name this node as the +//! target's authority. On `policy`, that's the policy of the acceptance +//! before, never the one landing ([`Policy::governing`]). Each of the +//! target's requirements needs a [claim](crate::annotations::Claim) +//! about the revision that lands: results with the same input digest +//! are promoted to it first, and a valid promotion claims what its +//! original does, as its original's issuer. +//! 4. **Missing evidence** leaves the landing waiting, until the statements +//! are issued and the landing is retried. +//! 5. **Otherwise** this node signs an acceptance naming what landed, the +//! evidence and the policy, and appends it to the target's chain. +//! +//! Landing another target works the same way: its tip is merged from the +//! newest acceptances both targets' histories share, and the requirements +//! apply to the merged snapshot. A merge that changes nothing lands +//! nothing if either tip is in the other's history: the target has the +//! source's already, or the source has nothing more. Otherwise it lands, +//! with no evidence then, so the target's history records the source's tip +//! and later merges start from it. +//! +//! The queue of each target lives in `queue//`: a `lock` held +//! while landing, so landings on one target never race, and a file for each +//! landing that waits, `change-` or `target-`, so waiting +//! landings can be listed and retried. + +use std::collections::HashSet; +use std::fmt; +use std::fs; +use std::io; +use std::path::PathBuf; + +use origin_platform::{DurableWrite, LockMode, LockedFile, Native, Platform}; +use origin_shared::{ + AcceptanceBody, AcceptanceId, Base, ChangeId, EncodedObject, Kind, NodeKey, Principal, + PrincipalName, Record, RecordId, RepoPath, Revision, RevisionId, Source, StatementId, TargetId, + TargetName, Tree, TreeId, +}; + +use crate::ancestry::Ancestry; +use crate::annotations::{AnnotateError, Annotations, Claim}; +use crate::conflict::Conflicts; +use crate::policy::{Policy, PolicyError, Requirement}; +use crate::store::{Store, StoreError}; +use crate::targets::Head; + +/// The landing queues of a store's targets, as run by one node. +#[derive(Debug)] +pub struct Queue<'a, P = Native> { + store: &'a Store

, + key: &'a NodeKey, + node: PrincipalName, +} + +/// What a landing lands. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Proposal { + /// A change, as of its head. + Change(ChangeId), + /// Another target, as of its tip. + Target(TargetName), +} + +/// How a landing went. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Landing { + /// It landed. + Landed { + /// What landed. + proposal: Proposal, + /// The revision or acceptance the acceptance names. + source: Source, + /// The acceptance. + acceptance: Head, + }, + /// The target already has it, so nothing landed. + Unchanged(Proposal), + /// It waits, and the queue keeps it. + Pending(Pending), +} + +/// A landing that waits, kept in the queue to be listed and retried. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Pending { + /// The target it lands on. + pub onto: TargetName, + /// What lands. + pub proposal: Proposal, + /// What the evidence must be about: the revision that would land, or + /// the merged snapshot of a target. A conflict is about the revision, + /// or the acceptance, that didn't merge. + pub subject: RecordId, + /// Why it waits. + pub blocked: Blocked, +} + +/// Why a landing waits. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Blocked { + /// The target requires these statements about the subject. + Evidence(Vec), + /// Both sides changed these paths. The `core.conflict` statement lists + /// them too. + Conflict { + /// The statement. + statement: StatementId, + /// The paths, sorted. + paths: Vec, + }, +} + +/// Landings onto one target, holding its queue's lock. +struct Session<'q, 'a, P> { + queue: &'q Queue<'a, P>, + onto: TargetName, + policy: Policy, + /// The acceptance of `policy` holding `policy`. + policy_head: AcceptanceId, + /// The target's latest acceptance, and its snapshot. + tip: Head, + state: TreeId, + annotations: Annotations<'a, P>, + _lock: LockedFile, +} + +/// What a session is about to land. +struct Candidate { + /// What lands. + proposal: Proposal, + /// What the queue keeps if it waits: the landing of the stack it's in. + kept_as: Proposal, + source: Source, + /// What the evidence must be about. + subject: RecordId, + /// The target's snapshot once it lands. + state: TreeId, + /// The trees the merge made, to write first. + trees: Vec, + /// The rebased revision, to write after the trees, if it is one. + revision: Option, +} + +impl<'a, P: Platform> Queue<'a, P> { + /// The queues of `store`, run by the node `node`, which signs with + /// `key`. + pub fn new(store: &'a Store

, key: &'a NodeKey, node: PrincipalName) -> Self { + Self { store, key, node } + } + + /// This node, which issues the statements and revisions the queue + /// makes. + fn principal(&self) -> Principal { + Principal::Node(self.node.clone()) + } + + /// Starts a repository's history: the `policy` target, holding `policy` + /// as its `policy.toml`, and an empty `main`, both of which the policy + /// must have this node order. Returns `main`'s first acceptance. + /// + /// # Errors + /// + /// Fails if the policy is invalid or doesn't make this node the + /// authority of both targets, either target exists, or the records + /// cannot be written. + pub fn init(&self, policy: &str) -> Result { + let (file, tree) = Policy::snapshot(policy); + let empty = Tree::default().encode(); + self.store.write_all(&[file, tree.clone(), empty.clone()])?; + let policy = Self::check_policy(self.store, tree.id().into())?; + for target in [TargetName::POLICY, TargetName::MAIN] { + self.authorize(&policy, &target)?; + } + let genesis = |target: &TargetName, state: &EncodedObject, policy| { + AcceptanceBody { + target: target.id(), + seq: 0, + prev: None, + state: state.id().into(), + landed: Vec::new(), + evidence: Vec::new(), + policy, + } + .sign(self.key) + }; + let policy = self.store.append( + &TargetName::POLICY, + &genesis(&TargetName::POLICY, &tree, None), + )?; + Ok(self.store.append( + &TargetName::MAIN, + &genesis(&TargetName::MAIN, &empty, Some(policy)), + )?) + } + + /// Lands `change`, and the changes below it that `onto` doesn't have + /// yet, from the bottom of the stack. Stops at the first that waits, + /// which the queue keeps as a landing of `change`: retrying it lands + /// the rest of the stack once that one can. + /// + /// # Errors + /// + /// Fails if this node isn't the target's authority, a change in the + /// stack diverged, was abandoned or has unresolved conflicts, or the + /// records cannot be read or written. + pub fn land_change( + &self, + change: ChangeId, + onto: &TargetName, + ) -> Result, LandError> { + let mut session = self.session(onto)?; + let stack = session.stack(change)?; + if stack.is_empty() { + session.forget(&Proposal::Change(change))?; + return Ok(vec![Landing::Unchanged(Proposal::Change(change))]); + } + let proposal = Proposal::Change(change); + let mut landings = Vec::new(); + for (id, revision) in stack { + let landing = session.land_revision(&proposal, id, revision)?; + let waits = matches!(landing, Landing::Pending(_)); + landings.push(landing); + if waits { + break; + } + } + Ok(landings) + } + + /// Lands the tip of `source` on `onto`. + /// + /// # Errors + /// + /// Fails if either target doesn't exist or they're the same, this node + /// isn't the authority of `onto`, or the records cannot be read or + /// written. + pub fn land_target( + &self, + source: &TargetName, + onto: &TargetName, + ) -> Result { + if source == onto { + return Err(LandError::SameTarget(onto.clone())); + } + let mut session = self.session(onto)?; + let tip = self + .store + .head(source)? + .ok_or_else(|| LandError::UnknownTarget(source.clone()))?; + let state = self.store.get(tip.acceptance)?.body().state; + let tips = [session.tip.acceptance, tip.acceptance]; + let history = session.history(&tips)?; + let base = history.merge_base(self.store, &tips)?; + let merge = self.store.merge(base, state, session.state)?; + let proposal = Proposal::Target(source.clone()); + if !merge.conflicts.is_empty() { + let subject = RecordId::Acceptance(tip.acceptance); + return session.block(proposal, subject, &merge.conflicts); + } + // A merge that changes nothing has nothing to record either when + // one tip is in the other's history: the target has the source's + // already, or the source has nothing the target lacks. + let [ours, theirs] = tips; + let either = history.descends(ours, theirs) || history.descends(theirs, ours); + if merge.tree == session.state && either { + session.forget(&proposal)?; + return Ok(Landing::Unchanged(proposal)); + } + // Even a merge that changes nothing lands, so that the tip is in + // the target's history, and later merges start from it. + session.submit(Candidate { + kept_as: proposal.clone(), + proposal, + source: Source::Target(tip.acceptance), + subject: RecordId::Snapshot(merge.tree), + state: merge.tree, + trees: merge.trees, + revision: None, + }) + } + + /// Creates the target `name` at the tip of `from`, with a first + /// acceptance landing it. + /// + /// # Errors + /// + /// Fails if `name` exists, `from` doesn't, this node wouldn't be the new + /// target's authority, or the records cannot be read or written. + pub fn create_target(&self, name: &TargetName, from: &TargetName) -> Result { + let _lock = self.lock(name)?; + if self.store.head(name)?.is_some() { + return Err(LandError::TargetExists(name.clone())); + } + let (policy_head, policy) = Policy::in_force(self.store)?; + self.authorize(&policy, name)?; + let from = self + .store + .head(from)? + .ok_or_else(|| LandError::UnknownTarget(from.clone()))?; + let acceptance = AcceptanceBody { + target: name.id(), + seq: 0, + prev: None, + state: self.store.get(from.acceptance)?.body().state, + landed: vec![Source::Target(from.acceptance)], + evidence: Vec::new(), + policy: Some(policy_head), + } + .sign(self.key); + let acceptance = self.store.append(name, &acceptance)?; + Ok(Head { seq: 0, acceptance }) + } + + /// Every landing that waits, on every target, sorted by target. + /// + /// # Errors + /// + /// Fails if the queues cannot be listed, or a landing cannot be read or + /// is malformed. + pub fn pending(store: &Store

) -> Result, StoreError> { + let root = store.queue_dir(); + let list = |dir: &PathBuf| match fs::read_dir(dir) { + Ok(entries) => entries + .map(|entry| entry.map(|entry| entry.path())) + .collect::>>() + .map_err(|source| StoreError::io(dir, source)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(Vec::new()), + Err(source) => Err(StoreError::io(dir, source)), + }; + let mut pending = Vec::new(); + for dir in list(&root)? { + for path in list(&dir)? { + let name = path.file_name().unwrap_or_default(); + if name == Self::LOCK || DurableWrite::

::is_temporary(name) { + continue; + } + // Read without the lock: a landing that just ended forgot it. + let text = match fs::read_to_string(&path) { + Ok(text) => text, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(source) => return Err(StoreError::io(&path, source)), + }; + let mut landing = + Pending::decode(&text).ok_or_else(|| StoreError::Malformed(path.clone()))?; + if let Blocked::Conflict { statement, paths } = &mut landing.blocked { + *paths = Conflicts::listed(store, &store.get(*statement)?)?.paths(); + } + pending.push(landing); + } + } + pending.sort_by(|a, b| a.onto.cmp(&b.onto)); + Ok(pending) + } + + /// The policy in `state`, about to become the policy in force, if it's + /// a valid one that names an authority for `policy`, so the target + /// never locks up. + fn check_policy(store: &Store

, state: TreeId) -> Result { + let policy = Policy::of_snapshot(store, state) + .and_then(|policy| policy.ok_or(PolicyError::Missing)) + .map_err(LandError::InvalidPolicy)?; + if policy.authority(&TargetName::POLICY).is_none() { + return Err(LandError::NoAuthority(TargetName::POLICY)); + } + Ok(policy) + } + + /// The name of the lock file in each target's queue. + const LOCK: &str = "lock"; + + /// The directory of `target`'s queue, created durably if it's missing. + fn queue_dir(&self, target: TargetId) -> Result { + let root = self.store.queue_dir(); + let dir = root.join(target.to_hex()); + if !dir.is_dir() { + fs::create_dir_all(&dir).map_err(|source| StoreError::io(&dir, source))?; + for parent in [&root, root.parent().unwrap_or(&root)] { + self.store.sync_dir(parent)?; + } + } + Ok(dir) + } + + /// Locks the queue of `target`, waiting for the landing in progress. + fn lock(&self, target: &TargetName) -> Result { + let path = self.queue_dir(target.id())?.join(Self::LOCK); + Ok(LockedFile::open( + &self.store.platform, + &path, + LockMode::Exclusive, + )?) + } + + /// Checks that this node is the authority of `target` under `policy`, + /// with the key it lists. + fn authorize(&self, policy: &Policy, target: &TargetName) -> Result<(), LandError> { + let authority = policy + .authority(target) + .ok_or_else(|| LandError::NoAuthority(target.clone()))?; + if *authority != self.node { + return Err(LandError::NotAuthority { + target: target.clone(), + authority: authority.clone(), + node: self.node.clone(), + }); + } + if policy.node_key(authority) != Some(&self.key.public()) { + return Err(LandError::WrongKey(self.node.clone())); + } + Ok(()) + } + + /// Starts landing on `onto`: locks its queue, then reads the policy in + /// force and the target's tip. + fn session(&self, onto: &TargetName) -> Result, LandError> { + let lock = self.lock(onto)?; + let tip = self + .store + .head(onto)? + .ok_or_else(|| LandError::UnknownTarget(onto.clone()))?; + // A landing on `policy` is governed by the acceptance before it: its + // tip, whatever the policy it lands says. + let (policy_head, policy) = if *onto == TargetName::POLICY { + let state = self.store.get(tip.acceptance)?.body().state; + (tip.acceptance, Self::check_policy(self.store, state)?) + } else { + Policy::in_force(self.store)? + }; + self.authorize(&policy, onto)?; + Ok(Session { + queue: self, + onto: onto.clone(), + policy, + policy_head, + state: self.store.get(tip.acceptance)?.body().state, + tip, + annotations: Annotations::open(self.store)?, + _lock: lock, + }) + } +} + +impl Session<'_, '_, P> { + fn store(&self) -> &Store

{ + self.queue.store + } + + /// The heads to land for `change`, bottom first: the change's head, and + /// below it, following each one's base, the heads of the changes the + /// target doesn't have yet. + fn stack(&self, change: ChangeId) -> Result, LandError> { + let index = self.annotations.index(); + let mut stack = Vec::new(); + let mut seen = HashSet::new(); + let mut next = Some(change); + while let Some(change) = next.take() { + if !seen.insert(change) { + break; + } + let head = match index.heads(change)[..] { + [head] => head, + [] => return Err(LandError::UnknownChange(change)), + ref heads => { + return Err(LandError::Divergent { + change, + heads: heads.to_vec(), + }); + } + }; + let revision: Revision = self.store().get(head)?; + if self.has_landed(head, &revision)? { + break; + } + let claims = self.annotations.claims(RecordId::Revision(head))?; + for claim in &claims { + let statement = claim.statement(); + if statement.kind == Kind::ABANDON && self.store().authorizes(statement)? { + return Err(LandError::Abandoned(change)); + } + } + let conflicts = Conflicts::of(self.store(), &claims, &revision)?; + if !conflicts.is_empty() { + return Err(LandError::Unresolved { + change, + paths: conflicts.paths(), + }); + } + if let Base::Revision(below) = revision.base { + next = Some(self.store().get(below)?.change); + } + stack.push((head, revision)); + } + stack.reverse(); + Ok(stack) + } + + /// Whether `revision`, `id`, landed on the target. A revision only lands + /// on the tip it's based on, so the acceptance after that tip tells. + fn has_landed(&self, id: RevisionId, revision: &Revision) -> Result { + let Base::Acceptance(base) = revision.base else { + return Ok(false); + }; + let base = self.store().get(base)?; + if base.body().target != self.onto.id() { + return Ok(false); + } + let Some(next) = self + .store() + .acceptance_at(&self.onto, base.body().seq + 1)? + else { + return Ok(false); + }; + Ok(self + .store() + .get(next)? + .body() + .landed + .contains(&Source::Revision(id))) + } + + /// Rebases the revision `id` onto the tip and lands it, as part of the + /// landing of `stack`, which is kept if it waits. + fn land_revision( + &mut self, + stack: &Proposal, + id: RevisionId, + revision: Revision, + ) -> Result { + let proposal = Proposal::Change(revision.change); + let base = match revision.base { + Base::Acceptance(id) => self.store().get(id)?.body().state, + Base::Revision(id) => self.store().get(id)?.snapshot, + }; + let merge = self + .store() + .merge(Some(base), revision.snapshot, self.state)?; + if !merge.conflicts.is_empty() { + return self.block(stack.clone(), RecordId::Revision(id), &merge.conflicts); + } + if merge.tree == self.state { + self.forget(&proposal)?; + return Ok(Landing::Unchanged(proposal)); + } + let tip = Base::Acceptance(self.tip.acceptance); + if revision.base == tip { + return self.submit(Candidate { + proposal, + kept_as: stack.clone(), + source: Source::Revision(id), + subject: RecordId::Revision(id), + state: merge.tree, + trees: merge.trees, + revision: None, + }); + } + let rebased = Revision { + predecessors: vec![id], + base: tip, + snapshot: merge.tree, + author: self.queue.principal(), + ..revision + } + .encode(); + let rebased_id = RevisionId::from(rebased.id()); + self.submit(Candidate { + proposal, + kept_as: stack.clone(), + source: Source::Revision(rebased_id), + subject: RecordId::Revision(rebased_id), + state: merge.tree, + trees: merge.trees, + revision: Some(rebased), + }) + } + + /// Checks the policy for `candidate`, and lands it if its evidence is + /// there, or keeps it waiting. Writes the records in layers, each + /// durable before the next refers to it: the trees, then the rebased + /// revision, then the promoted statements and the acceptance. + fn submit(&mut self, candidate: Candidate) -> Result { + let store = self.queue.store; + if !candidate.trees.is_empty() { + store.write_all(&candidate.trees)?; + } + // What a landing on `policy` lands governs the next one, which + // this node must still be the authority of. + let landed_policy = if self.onto == TargetName::POLICY { + self.queue.authorize(&self.policy, &self.onto)?; + Some(Queue::check_policy(store, candidate.state)?) + } else { + None + }; + let mut records: Vec = candidate.revision.into_iter().collect(); + // A landing that changes nothing, which only records another + // target's tip, needs no evidence. + let requirements = if candidate.state == self.state { + Vec::new() + } else { + self.policy.requirements(&self.onto).to_vec() + }; + let mut evidence = Vec::new(); + let mut missing = Vec::new(); + if !requirements.is_empty() { + // Promotion reads the revision. + store.write_all(&records)?; + records.clear(); + let promotions = self + .annotations + .promotions(candidate.subject, &self.queue.principal())?; + let mut claims = self.annotations.claims(candidate.subject)?; + for promotion in promotions { + let encoded = promotion.encode(); + let claim = store.claim(encoded.id().into(), promotion)?; + claims.push(claim.map_err(AnnotateError::from)?); + records.push(encoded); + } + for requirement in requirements { + match self.evidence(&claims, &requirement) { + Some(id) if !evidence.contains(&id) => evidence.push(id), + Some(_) => {} + None => missing.push(requirement), + } + } + } + if !missing.is_empty() { + store.write_all(&records)?; + let pending = Pending { + onto: self.onto.clone(), + proposal: candidate.kept_as, + subject: candidate.subject, + blocked: Blocked::Evidence(missing), + }; + self.keep(&pending)?; + return Ok(Landing::Pending(pending)); + } + + let acceptance = AcceptanceBody { + target: self.onto.id(), + seq: self.tip.seq + 1, + prev: Some(self.tip.acceptance), + state: candidate.state, + landed: vec![candidate.source], + evidence, + policy: Some(self.policy_head), + } + .sign(self.queue.key); + let encoded = acceptance.encode(); + if Policy::governing(encoded.id().into(), acceptance.body())? != self.policy_head { + return Err(PolicyError::NotPredecessor(acceptance.body().seq).into()); + } + records.push(encoded); + store.write_all(&records)?; + let id = store.append(&self.onto, &acceptance)?; + self.tip = Head { + seq: acceptance.body().seq, + acceptance: id, + }; + self.state = candidate.state; + if let Some(policy) = landed_policy { + self.policy = policy; + self.policy_head = id; + } + self.forget(&candidate.proposal)?; + Ok(Landing::Landed { + proposal: candidate.proposal, + source: candidate.source, + acceptance: self.tip, + }) + } + + /// The first of `claims` that meets `requirement`: of the kind and + /// verdict required, from an issuer the policy admits, which for a + /// promotion is its original's. + fn evidence(&self, claims: &[Claim], requirement: &Requirement) -> Option { + claims + .iter() + .find(|claim| { + let claimed = claim.claimed(); + claimed.kind == requirement.kind + && claimed.verdict == Some(requirement.verdict) + && self.policy.admits(&requirement.issuer, &claimed.issuer) + }) + .map(Claim::id) + } + + /// Blocks a landing on `conflicts`: issues a `core.conflict` statement + /// about `subject` listing them, unless this node's latest one says + /// the same, and keeps the landing waiting. + fn block( + &mut self, + proposal: Proposal, + subject: RecordId, + conflicts: &Conflicts, + ) -> Result { + let issuer = self.queue.principal(); + let current = self + .annotations + .current(subject, &Kind::CONFLICT, &issuer)?; + let (payload, conflict) = conflicts.statement(subject, issuer, current); + let unchanged = match current { + Some(id) => self.store().get(id)?.payload == conflict.payload, + None => false, + }; + let statement = match current { + Some(id) if unchanged => id, + _ => self.annotations.issue(&conflict, Some(payload))?, + }; + let pending = Pending { + onto: self.onto.clone(), + proposal, + subject, + blocked: Blocked::Conflict { + statement, + paths: conflicts.paths(), + }, + }; + self.keep(&pending)?; + Ok(Landing::Pending(pending)) + } + + /// The file keeping a waiting landing of `proposal`. + fn pending_path(&self, proposal: &Proposal) -> Result { + let name = match proposal { + Proposal::Change(change) => format!("change-{change}"), + Proposal::Target(target) => format!("target-{}", target.id()), + }; + Ok(self.queue.queue_dir(self.onto.id())?.join(name)) + } + + /// Durably keeps a waiting landing, replacing the last attempt's. + fn keep(&self, pending: &Pending) -> Result<(), StoreError> { + let write = self.store().durable_write(); + write.stage( + self.pending_path(&pending.proposal)?, + pending.encode().as_bytes(), + )?; + Ok(write.commit()?) + } + + /// Durably forgets a landing that waited, if one did. + fn forget(&self, proposal: &Proposal) -> Result<(), StoreError> { + let path = self.pending_path(proposal)?; + match fs::remove_file(&path) { + Ok(()) => self.store().sync_dir(path.parent().unwrap_or(&path)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(StoreError::io(&path, source)), + } + } + + /// The history of `tips`, the target's and another target's, to merge + /// that one from: the state of the newest acceptance both histories + /// share, or the [merge](Ancestry::merge_base) of several, if each + /// lacks another, as after targets landed each other. An acceptance's + /// history is the one before it, and the acceptances of other targets + /// it landed: a target's first lands the one it was created from, so + /// targets created from the same one share it. + /// + /// The nearest shared acceptance can be an older one, through a target + /// that forked early and landed recently, so both histories are read + /// whole, each acceptance once. + fn history(&self, tips: &[AcceptanceId]) -> Result, StoreError> { + Ancestry::walk(tips, |id| { + let acceptance = self.store().get(id)?; + let body = acceptance.body(); + let landed = body.landed.iter().filter_map(|source| match *source { + Source::Target(landed) => Some(landed), + Source::Revision(_) => None, + }); + Ok::<_, StoreError>((landed.chain(body.prev).collect(), body.state)) + }) + } +} + +impl Pending { + /// The first line of every file keeping a waiting landing. + const HEADER: &str = "origin pending 1"; + + /// Encodes the landing as lines: the header, `onto `, `change + /// ` or `target `, `subject `, then a `missing + /// ` line for each requirement it lacks, or + /// `conflict `, whose payload lists the paths. + fn encode(&self) -> String { + let mut lines = vec![Self::HEADER.to_owned(), format!("onto {}", self.onto)]; + lines.push(match &self.proposal { + Proposal::Change(change) => format!("change {change}"), + Proposal::Target(target) => format!("target {target}"), + }); + let (kind, id) = match self.subject { + RecordId::Snapshot(id) => ("snapshot", id.to_hex()), + RecordId::Revision(id) => ("revision", id.to_hex()), + RecordId::Statement(id) => ("statement", id.to_hex()), + RecordId::Acceptance(id) => ("acceptance", id.to_hex()), + }; + lines.push(format!("subject {kind} {id}")); + match &self.blocked { + Blocked::Evidence(missing) => lines.extend(missing.iter().map(|requirement| { + format!( + "missing {} {} {}", + requirement.kind, requirement.verdict, requirement.issuer + ) + })), + Blocked::Conflict { statement, .. } => lines.push(format!("conflict {statement}")), + } + lines.join("\n") + "\n" + } + + /// Decodes what [`Pending::encode`] wrote, without the conflicting + /// paths, or `None` if it's malformed. + fn decode(text: &str) -> Option { + let mut lines = text.lines(); + if lines.next()? != Self::HEADER { + return None; + } + let onto = lines.next()?.strip_prefix("onto ")?.parse().ok()?; + let proposal = match lines.next()?.split_once(' ')? { + ("change", id) => Proposal::Change(id.parse().ok()?), + ("target", name) => Proposal::Target(name.parse().ok()?), + _ => return None, + }; + let subject = match lines.next()?.strip_prefix("subject ")?.split_once(' ')? { + ("snapshot", id) => RecordId::Snapshot(id.parse().ok()?), + ("revision", id) => RecordId::Revision(id.parse().ok()?), + ("statement", id) => RecordId::Statement(id.parse().ok()?), + ("acceptance", id) => RecordId::Acceptance(id.parse().ok()?), + _ => return None, + }; + let mut missing = Vec::new(); + let mut conflict = None; + for line in lines { + match line.split_once(' ')? { + ("missing", requirement) => { + let mut fields = requirement.split(' '); + missing.push(Requirement { + kind: fields.next()?.parse().ok()?, + verdict: fields.next()?.parse().ok()?, + issuer: fields.next()?.parse().ok()?, + }); + } + ("conflict", id) => conflict = Some(id.parse().ok()?), + _ => return None, + } + } + let blocked = match conflict { + Some(statement) => Blocked::Conflict { + statement, + paths: Vec::new(), + }, + None => Blocked::Evidence(missing), + }; + Some(Self { + onto, + proposal, + subject, + blocked, + }) + } +} + +impl fmt::Display for Proposal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Change(change) => write!(f, "change {}", change.short()), + Self::Target(target) => write!(f, "`{target}`"), + } + } +} + +/// A failure to land. +#[derive(Debug, thiserror::Error)] +pub enum LandError { + /// No target has this name. + #[error("no target is named `{0}`")] + UnknownTarget(TargetName), + /// `target new` was given a name already taken. + #[error("a target is already named `{0}`")] + TargetExists(TargetName), + /// A target was landed on itself. + #[error("cannot land `{0}` on itself")] + SameTarget(TargetName), + /// The policy names no authority for the target. + #[error("policy.toml names no authority for `{0}`: give it, or `policy`, a [targets] table")] + NoAuthority(TargetName), + /// Another node orders the target. + #[error( + "node:{authority} is the authority of `{target}`, so only it lands there, and this is \ + node:{node}" + )] + NotAuthority { + /// The target. + target: TargetName, + /// Its authority. + authority: PrincipalName, + /// This node. + node: PrincipalName, + }, + /// The policy lists another key for this node. + #[error("policy.toml lists another key for node:{0} than this node's")] + WrongKey(PrincipalName), + /// No revision of the change is known. + #[error("change {} has no revisions", .0.short())] + UnknownChange(ChangeId), + /// The change has several heads. + #[error( + "change {} is divergent, so it cannot land: its heads are {}. Run `origin converge {}` \ + first", + change.short(), + heads.iter().map(RevisionId::short).collect::>().join(", "), + change.short() + )] + Divergent { + /// The change. + change: ChangeId, + /// Its heads. + heads: Vec, + }, + /// The change was given up. + #[error("change {} was abandoned, so it cannot land", .0.short())] + Abandoned(ChangeId), + /// The change's head has unresolved conflicts. + #[error( + "change {} has unresolved conflicts, so it cannot land: edit {}, or accept them as they \ + are with `origin resolve`", + change.short(), + paths.iter().map(ToString::to_string).collect::>().join(", ") + )] + Unresolved { + /// The change. + change: ChangeId, + /// The unresolved paths. + paths: Vec, + }, + /// Landing on `policy` would leave it without a valid policy. + #[error("landing would leave `policy` without a valid policy.toml")] + InvalidPolicy(#[source] PolicyError), + /// The policy in force cannot be read. + #[error(transparent)] + Policy(#[from] PolicyError), + /// A conflict cannot be recorded. + #[error(transparent)] + Annotate(#[from] AnnotateError), + /// Records cannot be read or written. + #[error(transparent)] + Store(#[from] StoreError), +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + use std::thread; + + use origin_shared::{BlobId, Digest, Inputs, Statement, Verdict}; + + use super::*; + use crate::conflict::Conflicted; + use crate::tree::tests::build; + + /// A repository whose `policy` and `main` the node `queue` orders, with + /// `extra` added to its `policy.toml`. + struct Repo { + _dir: tempfile::TempDir, + store: Store, + key: NodeKey, + } + + impl Repo { + fn new(extra: &str) -> Self { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let key = NodeKey::from_bytes(&[7; 32]); + let repo = Self { + _dir: dir, + store, + key, + }; + repo.queue().init(&repo.policy_toml(extra)).unwrap(); + repo + } + + /// A `policy.toml` naming this node the authority of `policy`, then + /// `extra`. + fn policy_toml(&self, extra: &str) -> String { + format!( + "[nodes]\nqueue = \"{}\"\n[targets.policy]\nauthority = \"node:queue\"\n{extra}", + self.key.public() + ) + } + + /// Writes a snapshot of `files`, contents included. + fn files(&self, files: &[(&str, &str)]) -> TreeId { + let blobs: Vec<_> = files + .iter() + .map(|(_, content)| EncodedObject::blob(content.as_bytes())) + .collect(); + self.store.write_all(&blobs).unwrap(); + build(&self.store, files) + } + + fn queue(&self) -> Queue<'_> { + Queue::new(&self.store, &self.key, PrincipalName::new("queue").unwrap()) + } + + fn tip(&self, target: &TargetName) -> Head { + self.store.head(target).unwrap().unwrap() + } + + fn state(&self, target: &TargetName) -> TreeId { + self.store + .get(self.tip(target).acceptance) + .unwrap() + .body() + .state + } + + /// Writes a revision of `change` with `files`, made on `base`. + fn revision( + &self, + change: u8, + predecessors: &[RevisionId], + base: Base, + files: &[(&str, &str)], + ) -> RevisionId { + let encoded = Revision { + change: ChangeId::from_bytes([change; 16]), + predecessors: predecessors.to_vec(), + base, + deps: Vec::new(), + snapshot: self.files(files), + description: BlobId::from(EncodedObject::blob(b"").id()), + author: "user:ada".parse().unwrap(), + } + .encode(); + self.store + .write_all(std::slice::from_ref(&encoded)) + .unwrap(); + encoded.id().into() + } + + /// A revision of a new change `change` on the tip of `main`, adding + /// `files` to it. + fn change_on_main(&self, change: u8, files: &[(&str, &str)]) -> RevisionId { + let base = Base::Acceptance(self.tip(&TargetName::MAIN).acceptance); + let mut all = self.main_files(); + all.extend( + files + .iter() + .map(|&(path, content)| (path.to_owned(), content.to_owned())), + ); + let all: Vec<_> = all.iter().map(|(p, c)| (p.as_str(), c.as_str())).collect(); + self.revision(change, &[], base, &all) + } + + /// The files of `main`'s tip, with their contents. + fn main_files(&self) -> Vec<(String, String)> { + let empty = build(&self.store, &[]); + self.store + .diff(empty, self.state(&TargetName::MAIN)) + .unwrap() + .into_iter() + .map(|change| { + let content = self + .store + .try_blob(change.new.unwrap().id) + .unwrap() + .unwrap(); + ( + change.path.as_str().to_owned(), + String::from_utf8(content).unwrap(), + ) + }) + .collect() + } + + fn land(&self, change: u8) -> Vec { + self.queue() + .land_change(ChangeId::from_bytes([change; 16]), &TargetName::MAIN) + .unwrap() + } + } + + fn change(id: u8) -> Proposal { + Proposal::Change(ChangeId::from_bytes([id; 16])) + } + + /// The revision a landing landed. + fn landed(landing: &Landing) -> RevisionId { + match landing { + Landing::Landed { + source: Source::Revision(revision), + .. + } => *revision, + other => panic!("expected a landed revision, got {other:?}"), + } + } + + #[test] + fn lands_rebases_and_skips_what_the_target_has() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let a = repo.change_on_main(1, &[("a", "a")]); + let b = repo.change_on_main(2, &[("b", "b")]); + + let [landing] = &repo.land(1)[..] else { + panic!() + }; + assert_eq!(landed(landing), a, "made on the tip, it lands as it is"); + let acceptance = repo + .store + .get(repo.tip(&TargetName::MAIN).acceptance) + .unwrap(); + acceptance.verify(&repo.key.public()).unwrap(); + assert_eq!( + acceptance.body().policy, + Some(repo.tip(&TargetName::POLICY).acceptance) + ); + + let [landing] = &repo.land(2)[..] else { + panic!() + }; + let rebased = landed(landing); + assert_ne!(rebased, b, "made on an older tip, it is rebased"); + let revision = repo.store.get(rebased).unwrap(); + assert_eq!(revision.predecessors, [b]); + assert_eq!(revision.author, "node:queue".parse().unwrap()); + let log = repo.store.log(&TargetName::MAIN).unwrap(); + assert_eq!( + revision.base, + Base::Acceptance(log[1]), + "on the tip it landed after" + ); + assert_eq!( + repo.state(&TargetName::MAIN), + repo.files(&[("a", "a"), ("b", "b")]) + ); + + // Both changes are on `main` now: neither lands again, though `b`'s + // head is the rebased revision and `a`'s the original. + assert_eq!(repo.land(1), [Landing::Unchanged(change(1))]); + assert_eq!(repo.land(2), [Landing::Unchanged(change(2))]); + assert_eq!(repo.tip(&TargetName::MAIN).seq, 2); + } + + #[test] + fn stacks_land_from_the_bottom() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let below = repo.change_on_main(1, &[("a", "a")]); + let above = repo.revision(2, &[], Base::Revision(below), &[("a", "a"), ("b", "b")]); + let landings = repo.land(2); + assert_eq!(landings.len(), 2, "{landings:?}"); + assert_eq!(landed(&landings[0]), below); + assert_ne!(landed(&landings[1]), above, "rebased onto the first"); + assert_eq!( + repo.state(&TargetName::MAIN), + repo.files(&[("a", "a"), ("b", "b")]) + ); + } + + #[test] + fn conflicts_block_until_a_revision_merges() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + repo.change_on_main(1, &[("shared", "one")]); + let two = repo.change_on_main(2, &[("shared", "two")]); + repo.land(1); + + let blocked = repo.land(2); + let [Landing::Pending(pending)] = &blocked[..] else { + panic!("{blocked:?}") + }; + assert_eq!(pending.subject, RecordId::Revision(two)); + let Blocked::Conflict { statement, paths } = &pending.blocked else { + panic!("{pending:?}") + }; + assert_eq!(paths, &[RepoPath::new("shared").unwrap()]); + let conflict = repo.store.get(*statement).unwrap(); + assert_eq!(conflict.kind, Kind::CONFLICT); + assert_eq!(conflict.subject, RecordId::Revision(two)); + assert_eq!(repo.land(2), blocked, "retrying issues the same statement"); + assert_eq!( + Queue::pending(&repo.store).unwrap(), + std::slice::from_ref(pending) + ); + + // A new revision that takes `main`'s version merges. + let tip = Base::Acceptance(repo.tip(&TargetName::MAIN).acceptance); + repo.revision(2, &[two], tip, &[("shared", "one"), ("more", "m")]); + assert!(matches!(repo.land(2)[..], [Landing::Landed { .. }])); + assert_eq!(Queue::pending(&repo.store).unwrap(), []); + } + + /// A `ci.test` result about `subject` from `issuer`, reading `src`. + fn ci_test(subject: RevisionId, issuer: &str) -> Statement { + Statement { + subject: RecordId::Revision(subject), + kind: Kind::new("ci.test").unwrap(), + verdict: Some(Verdict::Pass), + issuer: issuer.parse().unwrap(), + inputs: Some(Inputs { + paths: vec![RepoPath::new("src").unwrap()], + definition: Digest::of(b"cargo test"), + toolchain: Digest::of(b"rustc"), + external: false, + }), + payload: None, + supersedes: None, + promoted_from: None, + } + } + + #[test] + fn missing_evidence_waits_and_promoted_results_count() { + let repo = Repo::new( + "[groups]\nrunners = [\"node:ci\"]\n[targets.main]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"ci.test\", verdict = \"pass\", issuer = \"group:runners\" }]\n", + ); + let code = repo.change_on_main(1, &[("src/lib.rs", "fn a() {}")]); + let docs = repo.change_on_main(2, &[("docs/guide.md", "# Guide")]); + let waiting = repo.land(1); + let [Landing::Pending(pending)] = &waiting[..] else { + panic!("{waiting:?}") + }; + assert!(matches!(&pending.blocked, Blocked::Evidence(missing) if missing.len() == 1)); + assert_eq!( + Queue::pending(&repo.store).unwrap(), + std::slice::from_ref(pending) + ); + + // A user outside the group doesn't count; the runner does. + let annotations = Annotations::open(&repo.store).unwrap(); + annotations + .issue(&ci_test(code, "user:mallory"), None) + .unwrap(); + assert_eq!(repo.land(1), waiting); + let result = annotations.issue(&ci_test(code, "node:ci"), None).unwrap(); + let [landing] = &repo.land(1)[..] else { + panic!() + }; + let acceptance = repo + .store + .get(repo.tip(&TargetName::MAIN).acceptance) + .unwrap(); + assert_eq!(landed(landing), code); + assert_eq!(acceptance.body().evidence, [result]); + assert_eq!(Queue::pending(&repo.store).unwrap(), []); + + // The docs change is rebased onto `src` as tested, so the result is + // promoted to it, and counts as the runner's. + let [landing] = &repo.land(2)[..] else { + panic!() + }; + let rebased = landed(landing); + assert_ne!(rebased, docs); + let acceptance = repo + .store + .get(repo.tip(&TargetName::MAIN).acceptance) + .unwrap(); + let [evidence] = acceptance.body().evidence[..] else { + panic!() + }; + let promoted = repo.store.get(evidence).unwrap(); + assert_eq!(promoted.subject, RecordId::Revision(rebased)); + assert_eq!(promoted.promoted_from, Some(result)); + assert_eq!(promoted.issuer, "node:queue".parse().unwrap()); + } + + #[test] + fn refuses_divergent_and_abandoned_changes() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let first = repo.change_on_main(1, &[("a", "a")]); + let base = Base::Acceptance(repo.tip(&TargetName::MAIN).acceptance); + repo.revision(1, &[first], base, &[("a", "left")]); + repo.revision(1, &[first], base, &[("a", "right")]); + let main = TargetName::MAIN; + let error = repo + .queue() + .land_change(ChangeId::from_bytes([1; 16]), &main) + .unwrap_err(); + assert!(matches!(error, LandError::Divergent { .. }), "{error}"); + + let given_up = repo.change_on_main(2, &[("b", "b")]); + let abandon = Statement { + subject: RecordId::Revision(given_up), + kind: Kind::ABANDON, + verdict: None, + issuer: "user:ada".parse().unwrap(), + inputs: None, + payload: None, + supersedes: None, + promoted_from: None, + }; + repo.store.write_all(&[abandon.encode()]).unwrap(); + let error = repo + .queue() + .land_change(ChangeId::from_bytes([2; 16]), &main) + .unwrap_err(); + assert!(matches!(error, LandError::Abandoned(_)), "{error}"); + } + + #[test] + fn only_the_authority_lands() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + repo.change_on_main(1, &[("a", "a")]); + let change = ChangeId::from_bytes([1; 16]); + let other = Queue::new(&repo.store, &repo.key, PrincipalName::new("other").unwrap()); + let error = other.land_change(change, &TargetName::MAIN).unwrap_err(); + assert!(matches!(error, LandError::NotAuthority { .. }), "{error}"); + + let impostor = NodeKey::from_bytes(&[8; 32]); + let queue = Queue::new(&repo.store, &impostor, PrincipalName::new("queue").unwrap()); + let error = queue.land_change(change, &TargetName::MAIN).unwrap_err(); + assert!(matches!(error, LandError::WrongKey(_)), "{error}"); + assert_eq!(repo.tip(&TargetName::MAIN).seq, 0); + } + + #[test] + fn concurrent_landings_serialize() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let (proposers, each) = (6_u8, 3_u8); + let changes: Vec = (0..proposers * each).map(|change| change + 1).collect(); + for &change in &changes { + repo.change_on_main(change, &[(&format!("file-{change}"), "x")]); + } + let landed = Mutex::new(0); + thread::scope(|scope| { + for chunk in changes.chunks(usize::from(each)) { + let (repo, landed) = (&repo, &landed); + scope.spawn(move || { + for &change in chunk { + assert!(matches!(repo.land(change)[..], [Landing::Landed { .. }])); + *landed.lock().unwrap() += 1; + } + }); + } + }); + assert_eq!(*landed.lock().unwrap(), changes.len()); + let log = repo.store.log(&TargetName::MAIN).unwrap(); + assert_eq!(log.len(), changes.len() + 1); + for pair in log.windows(2) { + assert_eq!(repo.store.get(pair[1]).unwrap().body().prev, Some(pair[0])); + } + assert_eq!(repo.main_files().len(), changes.len(), "every file landed"); + } + + #[test] + fn targets_land_on_each_other() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let (main, overhaul) = (TargetName::MAIN, TargetName::new("overhaul").unwrap()); + repo.change_on_main(1, &[("shared", "v1")]); + repo.land(1); + let queue = repo.queue(); + queue.create_target(&overhaul, &main).unwrap(); + assert!(matches!( + queue.create_target(&overhaul, &main), + Err(LandError::TargetExists(_)) + )); + assert_eq!(repo.state(&overhaul), repo.state(&main)); + + // Work lands on the overhaul while `main` moves on elsewhere. + let on_overhaul = Base::Acceptance(repo.tip(&overhaul).acceptance); + repo.revision(2, &[], on_overhaul, &[("shared", "v2"), ("new", "n")]); + queue + .land_change(ChangeId::from_bytes([2; 16]), &overhaul) + .unwrap(); + repo.change_on_main(3, &[("other", "o")]); + repo.land(3); + + // Catching up merges `main`'s change since the fork; finishing + // lands the overhaul on `main`. + assert!(matches!( + queue.land_target(&main, &overhaul).unwrap(), + Landing::Landed { .. } + )); + let landing = queue.land_target(&overhaul, &main).unwrap(); + assert!(matches!( + landing, + Landing::Landed { + source: Source::Target(_), + .. + } + )); + let expected = repo.files(&[("shared", "v2"), ("new", "n"), ("other", "o")]); + assert_eq!(repo.state(&main), expected); + assert_eq!(repo.state(&overhaul), expected); + assert_eq!( + queue.land_target(&overhaul, &main).unwrap(), + Landing::Unchanged(Proposal::Target(overhaul.clone())) + ); + assert!(matches!( + queue.land_target(&main, &main), + Err(LandError::SameTarget(_)) + )); + } + + #[test] + fn policy_landings_keep_a_valid_policy() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let policy = TargetName::POLICY; + let tip = Base::Acceptance(repo.tip(&policy).acceptance); + for (change, toml) in [(1, "not = [valid"), (2, "[nodes]\n")] { + repo.revision(change, &[], tip, &[("policy.toml", toml)]); + let result = repo + .queue() + .land_change(ChangeId::from_bytes([change; 16]), &policy); + assert!( + matches!( + result, + Err(LandError::InvalidPolicy(_) | LandError::NoAuthority(_)) + ), + "{result:?}" + ); + } + // A valid edit lands, and later landings are under it. + let requiring = repo.policy_toml( + "[targets.main]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"review\", verdict = \"pass\", issuer = \"user:bob\" }]\n", + ); + let encoded = Revision { + change: ChangeId::from_bytes([3; 16]), + predecessors: Vec::new(), + base: tip, + deps: Vec::new(), + snapshot: repo.files(&[("policy.toml", &requiring)]), + description: BlobId::from(EncodedObject::blob(b"").id()), + author: "user:ada".parse().unwrap(), + } + .encode(); + repo.store + .write_all(std::slice::from_ref(&encoded)) + .unwrap(); + let landings = repo + .queue() + .land_change(ChangeId::from_bytes([3; 16]), &policy) + .unwrap(); + assert!( + matches!(landings[..], [Landing::Landed { .. }]), + "{landings:?}" + ); + repo.change_on_main(4, &[("a", "a")]); + assert!(matches!(repo.land(4)[..], [Landing::Pending(_)])); + } + + #[test] + fn pending_landings_round_trip() { + let pending = Pending { + onto: TargetName::new("release/v3").unwrap(), + proposal: Proposal::Target(TargetName::MAIN), + subject: RecordId::Snapshot(Tree::default().encode().id().into()), + blocked: Blocked::Evidence(vec![Requirement { + kind: Kind::new("ci.test").unwrap(), + verdict: Verdict::Pass, + issuer: "group:runners".parse().unwrap(), + }]), + }; + assert_eq!(Pending::decode(&pending.encode()), Some(pending.clone())); + let conflict = Pending { + proposal: change(9), + blocked: Blocked::Conflict { + statement: EncodedObject::blob(b"conflict").id().into(), + paths: Vec::new(), + }, + ..pending + }; + assert_eq!(Pending::decode(&conflict.encode()), Some(conflict)); + assert_eq!(Pending::decode("origin pending 2\n"), None); + } + + /// A repository whose `main` requires a passing `ci.test` from + /// `user:ci`. + fn requiring_ci() -> Repo { + Repo::new( + "[targets.main]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"ci.test\", verdict = \"pass\", issuer = \"user:ci\" }]\n", + ) + } + + #[test] + fn forged_promotions_are_no_evidence() { + let repo = requiring_ci(); + let failed = repo.change_on_main(1, &[("src/lib.rs", "bad")]); + let mut original = ci_test(failed, "user:ci"); + original.verdict = Some(Verdict::Fail); + let id = Annotations::open(&repo.store) + .unwrap() + .issue(&original, None) + .unwrap(); + // Eve wraps the trusted failure as a pass, written as a fetch would. + let forged = Statement { + verdict: Some(Verdict::Pass), + issuer: "user:eve".parse().unwrap(), + promoted_from: Some(id), + ..original.clone() + }; + // And carries a result that read the network to another change. + let old = repo.change_on_main(3, &[("src/lib.rs", "bad")]); + let docs = repo.change_on_main(2, &[("src/lib.rs", "bad"), ("docs", "d")]); + let mut external = ci_test(old, "user:ci"); + external.inputs.as_mut().unwrap().external = true; + let external_id: StatementId = external.encode().id().into(); + let reused = Statement { + subject: RecordId::Revision(docs), + issuer: "user:eve".parse().unwrap(), + promoted_from: Some(external_id), + ..external.clone() + }; + repo.store.write_all(&[external.encode()]).unwrap(); + repo.store + .write_all(&[forged.encode(), reused.encode()]) + .unwrap(); + assert!(matches!(repo.land(1)[..], [Landing::Pending(_)])); + assert!(matches!(repo.land(2)[..], [Landing::Pending(_)])); + } + + #[test] + fn replaced_results_are_promoted_again() { + let repo = requiring_ci(); + let old = repo.change_on_main(1, &[("src/lib.rs", "code")]); + let new = repo.change_on_main(2, &[("src/lib.rs", "code"), ("docs", "new")]); + let original = ci_test(old, "user:ci"); + let annotations = Annotations::open(&repo.store).unwrap(); + let id = annotations.issue(&original, None).unwrap(); + Annotations::open(&repo.store) + .unwrap() + .promote(RecordId::Revision(new), &"node:queue".parse().unwrap()) + .unwrap(); + // The same passing result runs again, replacing the one promoted. + let rerun = Statement { + supersedes: Some(id), + ..original + }; + annotations.issue(&rerun, None).unwrap(); + let landings = repo.land(2); + assert!( + matches!(landings[..], [Landing::Landed { .. }]), + "{landings:?}" + ); + } + + #[test] + fn only_the_creator_of_a_change_abandons_it() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let revision = repo.change_on_main(1, &[("file", "work")]); + let abandon = |issuer: &str| Statement { + subject: RecordId::Revision(revision), + kind: Kind::ABANDON, + verdict: None, + issuer: issuer.parse().unwrap(), + inputs: None, + payload: None, + supersedes: None, + promoted_from: None, + }; + repo.store + .write_all(&[abandon("user:eve").encode()]) + .unwrap(); + assert!(matches!(repo.land(1)[..], [Landing::Landed { .. }])); + } + + #[test] + fn unresolved_conflicts_never_land() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let marked = "<<<<<<< a\n=======\n>>>>>>> b\n"; + let revision = repo.change_on_main(1, &[("file", marked)]); + let path = RepoPath::new("file").unwrap(); + let conflicts = |content: &str| { + Conflicts::new(vec![Conflicted { + path: path.clone(), + written: Some(EncodedObject::blob(content.as_bytes()).id()), + }]) + }; + // From someone else, it says nothing about the revision's files. + let foreign = conflicts(marked).statement( + RecordId::Revision(revision), + "user:eve".parse().unwrap(), + None, + ); + // From its author, it does. + let (payload, statement) = conflicts(marked).statement( + RecordId::Revision(revision), + "user:ada".parse().unwrap(), + None, + ); + repo.store + .write_all(&[foreign.0, foreign.1.encode()]) + .unwrap(); + let mut annotations = Annotations::open(&repo.store).unwrap(); + assert_eq!( + annotations + .claims(RecordId::Revision(revision)) + .unwrap() + .len(), + 1 + ); + annotations.issue(&statement, Some(payload)).unwrap(); + annotations = Annotations::open(&repo.store).unwrap(); + assert_eq!( + annotations + .claims(RecordId::Revision(revision)) + .unwrap() + .len(), + 2 + ); + let error = repo + .queue() + .land_change(ChangeId::from_bytes([1; 16]), &TargetName::MAIN) + .unwrap_err(); + assert!( + matches!(&error, LandError::Unresolved { paths, .. } if *paths == [path.clone()]), + "{error}" + ); + + // A revision whose file is no longer what the conflict wrote there + // is resolved, whatever the file holds. + let edited = repo.change_on_main(2, &[("file", "Heading\n=======\n")]); + let (payload, statement) = conflicts(marked).statement( + RecordId::Revision(edited), + "user:ada".parse().unwrap(), + None, + ); + annotations.issue(&statement, Some(payload)).unwrap(); + assert!(matches!(repo.land(2)[..], [Landing::Landed { .. }])); + } + + #[test] + fn sibling_targets_merge_from_where_they_forked() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + repo.change_on_main(1, &[("keep", "k"), ("delete", "d")]); + repo.land(1); + let (a, b) = (TargetName::new("a").unwrap(), TargetName::new("b").unwrap()); + let queue = repo.queue(); + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + // `a` deletes a file, `b` adds one: neither ever landed the other. + let on = |target| Base::Acceptance(repo.tip(target).acceptance); + repo.revision(2, &[], on(&a), &[("keep", "k")]); + queue + .land_change(ChangeId::from_bytes([2; 16]), &a) + .unwrap(); + repo.revision( + 3, + &[], + on(&b), + &[("keep", "k"), ("delete", "d"), ("other", "o")], + ); + queue + .land_change(ChangeId::from_bytes([3; 16]), &b) + .unwrap(); + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!(repo.state(&b), repo.files(&[("keep", "k"), ("other", "o")])); + } + + #[test] + fn a_merge_that_changes_nothing_still_records_the_tip() { + // `b` requires tests, which a landing changing nothing doesn't. + let repo = Repo::new( + "[targets.b]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"ci.test\", verdict = \"pass\", issuer = \"user:ci\" }]\n", + ); + let queue = repo.queue(); + let annotations = Annotations::open(&repo.store).unwrap(); + // Results without inputs, which are never promoted to a merge. + let pass = |subject| { + let statement = Statement { + subject, + inputs: None, + ..ci_test(RevisionId::from(EncodedObject::blob(b"").id()), "user:ci") + }; + annotations.issue(&statement, None).unwrap(); + }; + let (a, b) = (TargetName::new("a").unwrap(), TargetName::new("b").unwrap()); + let land = |change: u8, target: &TargetName, files: &[(&str, &str)]| { + let on = Base::Acceptance(repo.tip(target).acceptance); + pass(RecordId::Revision(repo.revision(change, &[], on, files))); + queue + .land_change(ChangeId::from_bytes([change; 16]), target) + .unwrap(); + }; + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + // Both add the same file: `a`'s tip lands on `b`, changing nothing. + land(1, &a, &[("f", "f")]); + land(2, &b, &[("f", "f")]); + let state = repo.state(&b); + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!(repo.state(&b), state); + assert_eq!( + queue.land_target(&a, &b).unwrap(), + Landing::Unchanged(Proposal::Target(a.clone())) + ); + + // So once `a` deletes the file, landing it on `b` deletes it there. + land(3, &a, &[]); + let landing = queue.land_target(&a, &b).unwrap(); + let Landing::Pending(pending) = landing else { + panic!("a deletion waits for its evidence: {landing:?}") + }; + pass(pending.subject); + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!(repo.state(&b), repo.files(&[])); + + // A target and the one it was made from, neither changing: landing + // either on the other records nothing, however often. + let c = TargetName::new("c").unwrap(); + queue.create_target(&c, &TargetName::MAIN).unwrap(); + for _ in 0..2 { + for (from, onto) in [(&c, &TargetName::MAIN), (&TargetName::MAIN, &c)] { + assert_eq!( + queue.land_target(from, onto).unwrap(), + Landing::Unchanged(Proposal::Target(from.clone())) + ); + } + } + } + + #[test] + fn targets_merge_from_their_newest_shared_acceptance() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let queue = repo.queue(); + let [a, b, c, d] = ["a", "b", "c", "d"].map(|name| TargetName::new(name).unwrap()); + let land = |change: u8, target: &TargetName, files: &[(&str, &str)]| { + let on = Base::Acceptance(repo.tip(target).acceptance); + repo.revision(change, &[], on, files); + queue + .land_change(ChangeId::from_bytes([change; 16]), target) + .unwrap(); + }; + // `c` and `d` fork from an empty `main`, `a` and `b` from the next + // `main`, which adds a file. + queue.create_target(&c, &TargetName::MAIN).unwrap(); + queue.create_target(&d, &TargetName::MAIN).unwrap(); + repo.change_on_main(1, &[("delete", "d")]); + repo.land(1); + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + // `a` deletes it, `b` keeps it, and both move on. + for n in 0..4 { + let counter = n.to_string(); + land(10 + n, &a, &[("a", &counter)]); + land(20 + n, &b, &[("delete", "d"), ("b", &counter)]); + } + // Then each lands a side line: the empty `main` is nearer. + land(30, &c, &[("c", "c")]); + land(31, &d, &[("d", "d")]); + assert!(matches!( + queue.land_target(&c, &a).unwrap(), + Landing::Landed { .. } + )); + assert!(matches!( + queue.land_target(&d, &b).unwrap(), + Landing::Landed { .. } + )); + + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!( + repo.state(&b), + repo.files(&[("a", "3"), ("b", "3"), ("c", "c"), ("d", "d")]), + "the deletion stands" + ); + } + + #[test] + fn criss_crossed_targets_merge_what_their_bases_agree_on() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let queue = repo.queue(); + let [a, b] = ["a", "b"].map(|name| TargetName::new(name).unwrap()); + let land = |change: u8, target: &TargetName, files: &[(&str, &str)]| { + let on = Base::Acceptance(repo.tip(target).acceptance); + repo.revision(change, &[], on, files); + queue + .land_change(ChangeId::from_bytes([change; 16]), target) + .unwrap(); + }; + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + land(1, &a, &[("a", "0"), ("shared", "a")]); + land(2, &b, &[("b", "0"), ("shared", "b")]); + // Each lands the other's tip at once, resolving `shared` its own + // way: two merge bases, which disagree on it. + let tips = [repo.tip(&a), repo.tip(&b)]; + let policy = Some(repo.tip(&TargetName::POLICY).acceptance); + for (target, prev, other, shared) in + [(&a, tips[0], tips[1], "a"), (&b, tips[1], tips[0], "b")] + { + let merged = AcceptanceBody { + target: target.id(), + seq: prev.seq + 1, + prev: Some(prev.acceptance), + state: repo.files(&[("a", "0"), ("b", "0"), ("shared", shared)]), + landed: vec![Source::Target(other.acceptance)], + evidence: vec![], + policy, + } + .sign(&repo.key); + repo.store.append(target, &merged).unwrap(); + } + // Then each changes another file. + land(3, &a, &[("a", "1"), ("b", "0"), ("shared", "a")]); + land(4, &b, &[("a", "0"), ("b", "1"), ("shared", "b")]); + + let Landing::Pending(pending) = queue.land_target(&a, &b).unwrap() else { + panic!("the bases disagree on `shared`, so it can't be merged blindly"); + }; + assert!( + matches!(&pending.blocked, Blocked::Conflict { paths, .. } if paths.iter().map(RepoPath::as_str).eq(["shared"])), + "{pending:?}" + ); + // Once `a` takes `b`'s side, the disjoint edits merge. + land(5, &a, &[("a", "1"), ("b", "0"), ("shared", "b")]); + assert!(matches!( + queue.land_target(&a, &b).unwrap(), + Landing::Landed { .. } + )); + assert_eq!( + repo.state(&b), + repo.files(&[("a", "1"), ("b", "1"), ("shared", "b")]) + ); + } + + #[test] + fn bases_that_disagree_on_a_kind_conflict_on_any_change_under_it() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let queue = repo.queue(); + let land = |change: u8, target: &TargetName, files: &[(&str, &str)]| { + let on = Base::Acceptance(repo.tip(target).acceptance); + repo.revision(change, &[], on, files); + queue + .land_change(ChangeId::from_bytes([change; 16]), target) + .unwrap(); + }; + let [a, b] = ["a", "b"].map(|name| TargetName::new(name).unwrap()); + queue.create_target(&a, &TargetName::MAIN).unwrap(); + queue.create_target(&b, &TargetName::MAIN).unwrap(); + land(1, &a, &[("p", "file")]); + land(2, &b, &[("p/delete", "original")]); + // Each lands the other's tip at once, both choosing the directory: + // two merge bases, one with a file at `p`, the other a directory. + let tips = [repo.tip(&a), repo.tip(&b)]; + let policy = Some(repo.tip(&TargetName::POLICY).acceptance); + for (target, prev, other) in [(&a, tips[0], tips[1]), (&b, tips[1], tips[0])] { + let merged = AcceptanceBody { + target: target.id(), + seq: prev.seq + 1, + prev: Some(prev.acceptance), + state: repo.files(&[("p/delete", "original")]), + landed: vec![Source::Target(other.acceptance)], + evidence: vec![], + policy, + } + .sign(&repo.key); + repo.store.append(target, &merged).unwrap(); + } + // `a` deletes `p/delete`, `b` keeps it and adds beside it. + land(3, &a, &[("a", "new")]); + land(4, &b, &[("p/delete", "original"), ("p/new", "new")]); + + // What `p` was is unknown, so the deletion can't be told from an + // addition: it conflicts, rather than bring `p/delete` back. + let Landing::Pending(pending) = queue.land_target(&a, &b).unwrap() else { + panic!("the bases disagree on `p`, so it can't be merged blindly"); + }; + assert!( + matches!(&pending.blocked, Blocked::Conflict { paths, .. } if paths.iter().map(RepoPath::as_str).eq(["p"])), + "{pending:?}" + ); + } + + #[test] + fn a_waiting_stack_lands_whole_once_it_can() { + let repo = requiring_ci(); + let lower = repo.change_on_main(1, &[("a", "a")]); + repo.revision(2, &[], Base::Revision(lower), &[("a", "a"), ("b", "b")]); + let [Landing::Pending(pending)] = &repo.land(2)[..] else { + panic!("the lower change waits") + }; + assert_eq!(pending.subject, RecordId::Revision(lower)); + assert_eq!( + Queue::pending(&repo.store).unwrap(), + [Pending { + proposal: change(2), + ..pending.clone() + }], + "kept as the landing of the whole stack" + ); + + // Retrying it, as once the evidence arrives, lands both: the result + // is promoted to the upper change, which doesn't change `src`. + let annotations = Annotations::open(&repo.store).unwrap(); + annotations.issue(&ci_test(lower, "user:ci"), None).unwrap(); + let landings = repo.land(2); + assert_eq!(landed(&landings[0]), lower); + assert!( + matches!(landings[1], Landing::Landed { proposal: ref p, .. } if *p == change(2)), + "{landings:?}" + ); + assert_eq!(Queue::pending(&repo.store).unwrap(), []); + } + + #[test] + fn each_policy_landing_is_governed_by_the_one_before() { + let repo = Repo::new("[targets.main]\nauthority = \"node:queue\"\n"); + let policy = TargetName::POLICY; + // The first edit requires a review to land on `policy`; the second, + // stacked on it, must then wait for one. + let requiring = repo + .policy_toml("[targets.main]\nauthority = \"node:queue\"\n") + .replace( + "[targets.policy]\nauthority = \"node:queue\"\n", + "[targets.policy]\nauthority = \"node:queue\"\n\ + require = [{ kind = \"review\", verdict = \"pass\", issuer = \"user:bob\" }]\n", + ); + let first = repo.revision( + 1, + &[], + Base::Acceptance(repo.tip(&policy).acceptance), + &[("policy.toml", &requiring)], + ); + let open = repo.policy_toml("[targets.main]\nauthority = \"node:queue\"\n"); + repo.revision(2, &[], Base::Revision(first), &[("policy.toml", &open)]); + let landings = repo + .queue() + .land_change(ChangeId::from_bytes([2; 16]), &policy) + .unwrap(); + assert_eq!(landed(&landings[0]), first); + assert!(matches!(landings[1], Landing::Pending(_)), "{landings:?}"); + + let log = repo.store.log(&policy).unwrap(); + for (seq, &id) in log.iter().enumerate() { + let body = repo.store.get(id).unwrap().body().clone(); + let governing = Policy::governing(id, &body).unwrap(); + assert_eq!(governing, if seq == 0 { id } else { log[seq - 1] }); + } + // An acceptance naming any other policy than the one before it is + // refused, its own included. + let mut body = repo.store.get(log[1]).unwrap().body().clone(); + body.policy = Some(log[1]); + assert!(matches!( + Policy::governing(log[1], &body), + Err(PolicyError::NotPredecessor(1)) + )); + } +} diff --git a/apps/origin-src/src/store/mod.rs b/apps/origin-src/src/store/mod.rs index ed782a13..81861f1f 100644 --- a/apps/origin-src/src/store/mod.rs +++ b/apps/origin-src/src/store/mod.rs @@ -1,6 +1,7 @@ //! A repository's storage on disk, under its `.origin` directory: -//! content-addressed objects here, and target chains in -//! [`targets`](crate::targets). +//! content-addressed objects here, target chains in +//! [`targets`](crate::targets), and pending landings in +//! [`queue`](crate::queue). //! //! Each object lives in its own zstd-compressed file at //! `objects/loose//`. Objects are @@ -185,6 +186,10 @@ impl Store

{ self.root.join("index") } + pub(crate) fn queue_dir(&self) -> PathBuf { + self.root.join("queue") + } + pub(crate) fn path(&self, id: ObjectId) -> PathBuf { let hex = id.to_hex(); let mut path = self.loose(); diff --git a/apps/origin-src/src/targets.rs b/apps/origin-src/src/targets.rs index 05209177..a41eba14 100644 --- a/apps/origin-src/src/targets.rs +++ b/apps/origin-src/src/targets.rs @@ -20,6 +20,7 @@ use std::fmt; use std::fs::{self, File}; use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use std::slice; @@ -177,6 +178,32 @@ impl Store

{ Ok(log) } + /// The acceptance at position `seq` of `target`'s log, or `None` if + /// none is published there yet. Reads one line of the log. + /// + /// # Errors + /// + /// Fails if the target's files cannot be read or are malformed. + pub fn acceptance_at( + &self, + target: &TargetName, + seq: u64, + ) -> Result, StoreError> { + let files = self.target_files(target.id()); + let Some(head) = files.read_head()? else { + return Ok(None); + }; + if seq >= head.seq { + return Ok((seq == head.seq).then_some(head.acceptance)); + } + let path = files.log(); + let mut line = [0; TargetFiles::LINE]; + File::open(&path) + .and_then(|log| log.read_exact_at(&mut line, seq * TargetFiles::LINE as u64)) + .map_err(|source| StoreError::io(&path, source))?; + Ok(files.parse_log(&line)?.first().copied()) + } + /// The names of every target, in no particular order. /// /// # Errors @@ -363,6 +390,10 @@ mod tests { }) ); assert_eq!(store.log(&main).unwrap(), ids); + for (seq, &id) in (0..).zip(&ids) { + assert_eq!(store.acceptance_at(&main, seq).unwrap(), Some(id)); + } + assert_eq!(store.acceptance_at(&main, 3).unwrap(), None); let last = store.get(ids[2]).unwrap(); assert_eq!(last.body().prev, Some(ids[1])); last.verify(&key().public()).unwrap(); diff --git a/apps/origin-src/src/tree.rs b/apps/origin-src/src/tree.rs index 1e37cf63..cc59b90d 100644 --- a/apps/origin-src/src/tree.rs +++ b/apps/origin-src/src/tree.rs @@ -1,11 +1,15 @@ -//! Comparing trees, and looking up what they hold. +//! Comparing trees, merging them, and looking up what they hold. use std::collections::HashMap; use std::collections::hash_map::Entry; use origin_platform::Platform; -use origin_shared::{Digest, EntryKind, EntryName, Inputs, RepoPath, Tree, TreeEntry, TreeId}; +use origin_shared::{ + Digest, EncodedObject, EntryKind, EntryName, Inputs, ObjectId, Record, RepoPath, Tree, + TreeEntry, TreeId, +}; +use crate::conflict::{Conflicted, Conflicts}; use crate::store::{Store, StoreError}; /// A file that differs between two trees. @@ -82,6 +86,255 @@ impl Store

{ } } +/// Two trees merged with the changes each made to a common one. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Merge { + /// The merged tree: both sides' changes, and `ours` where they conflict. + pub tree: TreeId, + /// The trees the merge made, which the store may not have yet. + pub trees: Vec, + /// The files with conflict markers a [marked](Store::merge_marked) + /// merge made, which the store may not have yet either. + pub files: Vec, + /// The paths both sides changed, each its own way, with what the merge + /// left there. + pub conflicts: Conflicts, +} + +/// What a merge leaves where both sides changed a path, each its own way. +#[derive(Clone, Copy, Debug)] +enum OnConflict<'l> { + /// `ours`. + KeepOurs, + /// Both versions between conflict markers naming the sides with these + /// labels, see [`Conflicts::mark`]. + Mark([&'l str; 2]), + /// [`Store::UNKNOWN`], a version neither side can have. + Unknown, +} + +impl Store

{ + /// Merges `ours` and `theirs`, two trees made from `base`, file by file: + /// a path one side changed takes that side's version, and a path both + /// changed, each its own way, is a conflict, where the merge keeps + /// `ours`. A path is a file, so two sides that changed different files + /// of a directory merge cleanly. Without a base, every path either side + /// has counts as changed. Subtrees with equal ids are never read, so the + /// cost follows the size of the changes. + /// + /// # Errors + /// + /// Fails if a tree that differs cannot be read. + pub fn merge( + &self, + base: Option, + ours: TreeId, + theirs: TreeId, + ) -> Result { + self.merge_with(base, ours, theirs, OnConflict::KeepOurs) + } + + /// Merges like [`Store::merge`], but where both sides changed a text + /// file, the merged one holds both versions between conflict markers + /// naming the sides with `labels`, see [`Conflicts::mark`]. Binary files, + /// and paths a directory is on either side of, keep `ours`. + /// + /// # Errors + /// + /// Fails if a tree that differs, or a file that conflicts, cannot be + /// read. + pub fn merge_marked( + &self, + base: Option, + ours: TreeId, + theirs: TreeId, + labels: [&str; 2], + ) -> Result { + self.merge_with(base, ours, theirs, OnConflict::Mark(labels)) + } + + /// The entry a merge of merge bases has where they conflict: an id no + /// object has, so it equals neither side of a merge from it. A merge + /// never reads under it either: the bases may have had a directory + /// there, so any change at or under its path conflicts. + const UNKNOWN: TreeEntry = TreeEntry { + kind: EntryKind::File, + id: ObjectId::from_bytes([0; ObjectId::LEN]), + }; + + /// Merges like [`Store::merge`], but where both sides changed a path, + /// the merged tree has [`Store::UNKNOWN`]: how merge bases are merged + /// into one, so a merge from it conflicts there unless both sides + /// agree. + /// + /// # Errors + /// + /// Fails if a tree that differs cannot be read. + pub fn merge_bases( + &self, + base: Option, + ours: TreeId, + theirs: TreeId, + ) -> Result { + self.merge_with(base, ours, theirs, OnConflict::Unknown) + } + + fn merge_with( + &self, + base: Option, + ours: TreeId, + theirs: TreeId, + on_conflict: OnConflict<'_>, + ) -> Result { + let mut merge = Merge { + tree: ours, + trees: Vec::new(), + files: Vec::new(), + conflicts: Conflicts::default(), + }; + let merged = self.merge_dir( + &RepoPath::root(), + base, + Some(ours), + Some(theirs), + on_conflict, + &mut merge, + )?; + merge.tree = if let Some(tree) = merged { + tree + } else { + let empty = Tree::default().encode(); + let id = empty.id().into(); + merge.trees.push(empty); + id + }; + merge.trees.sort_unstable_by_key(EncodedObject::id); + merge.trees.dedup_by_key(|tree| tree.id()); + merge.files.sort_unstable_by_key(EncodedObject::id); + merge.files.dedup_by_key(|file| file.id()); + Ok(merge) + } + + /// Merges three versions of the directory `dir`, `None` standing for one + /// that doesn't exist, returning the merged one, or `None` if it's + /// empty. + fn merge_dir( + &self, + dir: &RepoPath, + base: Option, + ours: Option, + theirs: Option, + on_conflict: OnConflict<'_>, + merge: &mut Merge, + ) -> Result, StoreError> { + if ours == theirs || base == theirs { + return Ok(ours); + } + if base == ours { + return Ok(theirs); + } + let read = |id: Option| id.map(|id| self.get(id)).transpose(); + let (base, ours, theirs): (Tree, Tree, Tree) = ( + read(base)?.unwrap_or_default(), + read(ours)?.unwrap_or_default(), + read(theirs)?.unwrap_or_default(), + ); + let mut names: Vec<&EntryName> = [&base, &ours, &theirs] + .into_iter() + .flat_map(|tree| tree.iter().map(|(name, _)| name)) + .collect(); + names.sort_unstable(); + names.dedup(); + let mut merged = Tree::default(); + for name in names { + let [base, ours, theirs] = + [&base, &ours, &theirs].map(|tree| tree.get(name.as_str()).copied()); + let entry = if ours == theirs || base == theirs { + ours + } else if base == ours { + theirs + } else if base != Some(Self::UNKNOWN) + && ours.is_none_or(|ours| ours.kind.is_tree()) + && theirs.is_none_or(|theirs| theirs.kind.is_tree()) + { + // Both sides changed a directory, or one changed it and the + // other removed it: merge it file by file, unless what it was + // is unknown, which would take what one side removed for + // what the other added. + let subtree = |entry: Option| entry.and_then(TreeEntry::subtree); + self.merge_dir( + &dir.join(name), + subtree(base), + subtree(ours), + subtree(theirs), + on_conflict, + merge, + )? + .map(|tree| TreeEntry { + kind: EntryKind::Tree, + id: tree.into(), + }) + } else { + let written = match on_conflict { + OnConflict::KeepOurs => ours, + OnConflict::Mark(labels) => self.mark(ours, theirs, labels, merge)?, + OnConflict::Unknown => Some(Self::UNKNOWN), + }; + // Paths come in order, each once. + merge.conflicts.push(Conflicted { + path: dir.join(name), + written: written.map(|entry| entry.id), + }); + written + }; + if let Some(entry) = entry { + merged.insert(name.clone(), entry); + } + } + if merged.is_empty() { + return Ok(None); + } + let encoded = merged.encode(); + let id = encoded.id().into(); + merge.trees.push(encoded); + Ok(Some(id)) + } + + /// The entry of a file both sides changed, `ours` and `theirs`, that + /// holds both versions between conflict markers: a file on the side + /// that removed it is empty. Keeps `ours` if either side isn't a + /// regular file, or is binary or missing from the store. + fn mark( + &self, + ours: Option, + theirs: Option, + labels: [&str; 2], + merge: &mut Merge, + ) -> Result, StoreError> { + let content = |entry: Option| match entry { + None => Ok(Some(Vec::new())), + Some(entry) if matches!(entry.kind, EntryKind::File | EntryKind::Executable) => { + self.try_blob(entry.id) + } + Some(_) => Ok(None), + }; + let (Some(our_content), Some(their_content)) = (content(ours)?, content(theirs)?) else { + return Ok(ours); + }; + let Some(marked) = Conflicts::mark(&our_content, &their_content, labels) else { + return Ok(ours); + }; + let file = EncodedObject::blob(&marked); + let kind = ours.or(theirs).map_or(EntryKind::File, |entry| entry.kind); + let entry = TreeEntry { + kind, + id: file.id(), + }; + merge.files.push(file); + Ok(Some(entry)) + } +} + /// Looks up the entries of snapshots by path, reading each tree once, /// however many snapshots share it. #[derive(Debug)] @@ -271,6 +524,118 @@ pub(crate) mod tests { assert_eq!(store.diff(empty, old).unwrap().len(), 5); } + /// The files of `tree`, by path, with their contents' ids. + fn files(store: &Store, tree: TreeId) -> Vec<(String, ObjectId)> { + let empty = build(store, &[]); + store + .diff(empty, tree) + .unwrap() + .into_iter() + .map(|change| (change.path.as_str().to_owned(), change.new.unwrap().id)) + .collect() + } + + #[test] + fn merges_changes_to_different_files() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let base = build( + &store, + &[ + ("src/a", "a"), + ("src/b", "b"), + ("gone/x", "x"), + ("same", "s"), + ], + ); + // Ours edits a file and removes a directory; theirs edits another + // file of the same directory, adds one, and makes the same edit. + let ours = build(&store, &[("src/a", "a2"), ("src/b", "b"), ("same", "s2")]); + let theirs = build( + &store, + &[ + ("src/a", "a"), + ("src/b", "b2"), + ("gone/x", "x"), + ("new/deep", "n"), + ("same", "s2"), + ], + ); + let merge = store.merge(Some(base), ours, theirs).unwrap(); + assert_eq!(merge.conflicts, Conflicts::default()); + let expected = build( + &store, + &[ + ("src/a", "a2"), + ("src/b", "b2"), + ("new/deep", "n"), + ("same", "s2"), + ], + ); + assert_eq!(files(&store, merge.tree), files(&store, expected)); + assert_eq!(merge.tree, expected, "trees are canonical"); + assert!( + merge.trees.iter().any(|tree| tree.id() == expected.into()), + "the merged trees are returned to be written" + ); + + // Merging onto an unchanged base reads nothing and makes no tree. + let unchanged = store.merge(Some(base), ours, base).unwrap(); + assert_eq!((unchanged.tree, unchanged.trees), (ours, Vec::new())); + } + + #[test] + fn conflicts_keep_ours() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let base = build( + &store, + &[("edit", "1"), ("dir/removed", "r"), ("file", "f")], + ); + let ours = build( + &store, + &[("edit", "ours"), ("dir/removed", "r2"), ("file", "f")], + ); + let theirs = build( + &store, + &[("edit", "theirs"), ("file/now-dir", "d"), ("other", "o")], + ); + let merge = store.merge(Some(base), ours, theirs).unwrap(); + let paths: Vec<_> = merge + .conflicts + .paths() + .iter() + .map(ToString::to_string) + .collect(); + assert_eq!( + paths, + ["dir/removed", "edit"], + "both edited `edit`; ours edited what theirs removed" + ); + let expected = build( + &store, + &[ + ("edit", "ours"), + ("dir/removed", "r2"), + ("file/now-dir", "d"), + ("other", "o"), + ], + ); + assert_eq!(files(&store, merge.tree), files(&store, expected)); + + // Without a base, both sides added everything. + let unrelated = store.merge(None, ours, theirs).unwrap(); + let paths: Vec<_> = unrelated + .conflicts + .paths() + .iter() + .map(ToString::to_string) + .collect(); + assert_eq!(paths, ["edit", "file"]); + } + #[test] fn looks_entries_up_by_path() { let dir = tempfile::tempdir().unwrap(); diff --git a/apps/origin-src/src/workspace/annotate.rs b/apps/origin-src/src/workspace/annotate.rs index 210bd5a2..164a4972 100644 --- a/apps/origin-src/src/workspace/annotate.rs +++ b/apps/origin-src/src/workspace/annotate.rs @@ -93,7 +93,7 @@ impl Payload { impl Workspace { /// This node, which issues the statements core makes. - fn node_principal(&self) -> Principal { + pub(super) fn node_principal(&self) -> Principal { Principal::Node(self.config.node().clone()) } @@ -326,7 +326,7 @@ impl Workspace { let working_copy = self.refresh()?; let mut annotations = Annotations::open(&self.store)?; let revision = working_copy.find_revision(annotations.index(), rev)?; - let promoted = annotations.promote(revision, &self.node_principal())?; + let promoted = annotations.promote(RecordId::Revision(revision), &self.node_principal())?; if promoted.is_empty() { println!("Nothing to promote to revision {}", revision.short()); } diff --git a/apps/origin-src/src/workspace/land.rs b/apps/origin-src/src/workspace/land.rs new file mode 100644 index 00000000..a87fe7cb --- /dev/null +++ b/apps/origin-src/src/workspace/land.rs @@ -0,0 +1,334 @@ +//! The target commands: `land`, `target new` and `targets`, and starting a +//! change on a target with `new --on`. +//! +//! The queue runs in this process, as this node. After a landing, the +//! working copy follows its change: when it landed, or the target already +//! had it, the working copy starts a new change on the target's tip, so +//! later edits are made on what the target has; when the queue rebased +//! it, the working copy becomes the rebased revision; and when it sits on +//! a change that landed, it is rebased onto the tip, if that merges +//! cleanly. + +use origin_shared::{ + Base, ChangeId, EncodedObject, Record, RecordId, Revision, RevisionId, Source, TargetName, + TreeId, +}; + +use super::{WorkingCopy, Workspace, WorkspaceError}; +use crate::index::Index; +use crate::policy::Policy; +use crate::queue::{Blocked, Landing, Pending, Proposal, Queue}; + +impl Workspace { + /// `origin land [] [--onto ]`: lands a change, the + /// working copy's by default, with the changes below it the target + /// doesn't have yet, on the target its stack is on by default. + /// + /// # Errors + /// + /// Fails if the change cannot be found or cannot land, this node isn't + /// the target's authority, or the records cannot be read or written. + pub fn land( + &mut self, + change: Option<&str>, + onto: Option, + ) -> Result<(), WorkspaceError> { + let working_copy = self.refresh()?; + let index = Index::open(&self.store)?; + let change = working_copy.find_change(&index, change.unwrap_or("@"))?; + let onto = match onto { + Some(onto) => onto, + None => self.target_of(Self::single_head(&index, change)?)?, + }; + let key = self.config.node_key()?; + let landings = + Queue::new(&self.store, &key, self.config.node().clone()).land_change(change, &onto)?; + for landing in &landings { + self.print_landing(landing, &onto)?; + } + self.follow(&working_copy, &onto, &landings) + } + + /// `origin land --from --onto `: lands the tip of + /// another target. + /// + /// # Errors + /// + /// Fails if a target doesn't exist, this node isn't the authority of + /// `onto`, or the records cannot be read or written. + pub fn land_target(&self, from: &TargetName, onto: &TargetName) -> Result<(), WorkspaceError> { + let key = self.config.node_key()?; + let landing = + Queue::new(&self.store, &key, self.config.node().clone()).land_target(from, onto)?; + self.print_landing(&landing, onto) + } + + /// `origin land --status`: lists the landings that wait, and why. + /// + /// # Errors + /// + /// Fails if the queues cannot be read. + pub fn land_status(&self) -> Result<(), WorkspaceError> { + let pending = Queue::pending(&self.store)?; + if pending.is_empty() { + println!("No landing waits."); + } + for pending in &pending { + Self::print_pending(pending); + } + Ok(()) + } + + /// `origin target new --from `: creates a target at the + /// tip of another. + /// + /// # Errors + /// + /// Fails if the name is taken, `from` doesn't exist, this node wouldn't + /// be the target's authority, or the records cannot be written. + pub fn target_new(&self, name: &TargetName, from: &TargetName) -> Result<(), WorkspaceError> { + let key = self.config.node_key()?; + Queue::new(&self.store, &key, self.config.node().clone()).create_target(name, from)?; + let tip = self + .store + .head(from)? + .ok_or_else(|| WorkspaceError::UnknownTarget(from.clone()))?; + println!( + "Created target `{name}` at {}", + self.describe_acceptance(tip.acceptance)? + ); + Ok(()) + } + + /// `origin targets`: lists every target with its tip, its authority and + /// what landing on it requires. + /// + /// # Errors + /// + /// Fails if the targets or the policy cannot be read. + pub fn targets(&self) -> Result<(), WorkspaceError> { + let policy = Policy::load(&self.store)?; + let mut targets = self.store.targets()?; + targets.sort(); + for target in targets { + let Some(tip) = self.store.head(&target)? else { + continue; + }; + let authority = policy + .authority(&target) + .map_or_else(|| "no authority".to_owned(), |node| format!("node:{node}")); + println!( + "{target} #{} {} ({authority})", + tip.seq, + tip.acceptance.short() + ); + for requirement in policy.requirements(&target) { + println!(" requires {requirement}"); + } + } + Ok(()) + } + + /// `origin new [description] --on `: starts a change on the tip + /// of a target. + /// + /// # Errors + /// + /// Fails if the target doesn't exist, the working copy cannot be + /// snapshotted, or the change cannot be written or checked out. + pub fn new_on(&mut self, target: &TargetName, description: &str) -> Result<(), WorkspaceError> { + let working_copy = self.refresh()?; + self.start_on(target, description, working_copy.revision.snapshot) + } + + /// Starts a change described by `description` on the tip of `target`, + /// and checks it out over the files of `files`. + fn start_on( + &mut self, + target: &TargetName, + description: &str, + files: TreeId, + ) -> Result<(), WorkspaceError> { + let tip = self + .store + .head(target)? + .ok_or_else(|| WorkspaceError::UnknownTarget(target.clone()))?; + let description = EncodedObject::blob(description.as_bytes()); + let revision = Revision { + change: ChangeId::random().map_err(WorkspaceError::Random)?, + predecessors: Vec::new(), + base: Base::Acceptance(tip.acceptance), + deps: Vec::new(), + snapshot: self.store.get(tip.acceptance)?.body().state, + description: description.id().into(), + author: self.config.user(), + }; + let encoded = revision.encode(); + self.write(&[vec![description], vec![encoded.clone()]])?; + self.checkout(files, revision.snapshot)?; + self.point_at(encoded.id().into())?; + println!("Working copy: {}", self.headline(&revision)?); + println!("On top of: {}", self.describe_base(revision.base)?); + Ok(()) + } + + /// The target a revision's stack is on: that of the acceptance at its + /// bottom. + fn target_of(&self, revision: RevisionId) -> Result { + let mut base = self.store.get(revision)?.base; + loop { + match base { + Base::Revision(id) => base = self.store.get(id)?.base, + Base::Acceptance(id) => { + return self.target_name(self.store.get(id)?.body().target); + } + } + } + } + + /// Moves the working copy after `landings` on `onto`, as the module + /// explains. + fn follow( + &mut self, + working_copy: &WorkingCopy, + onto: &TargetName, + landings: &[Landing], + ) -> Result<(), WorkspaceError> { + let revision = &working_copy.revision; + let files = revision.snapshot; + // A change the target already had is as good as landed. + let landed = |change: ChangeId| { + landings.iter().any(|landing| match landing { + Landing::Landed { proposal, .. } | Landing::Unchanged(proposal) => { + *proposal == Proposal::Change(change) + } + Landing::Pending(_) => false, + }) + }; + if landed(revision.change) { + return self.start_on(onto, "", files); + } + for landing in landings { + if let Landing::Pending(Pending { + proposal: Proposal::Change(change), + subject: RecordId::Revision(rebased), + .. + }) = landing + && *change == revision.change + && *rebased != working_copy.id + { + let rebased = *rebased; + self.checkout(files, self.store.get(rebased)?.snapshot)?; + self.point_at(rebased)?; + println!( + "Working copy: {}", + self.headline(&self.store.get(rebased)?)? + ); + return Ok(()); + } + } + let Base::Revision(below) = revision.base else { + return Ok(()); + }; + let below = self.store.get(below)?; + if !landed(below.change) { + return Ok(()); + } + self.rebase_onto_tip(working_copy, below.snapshot, onto) + } + + /// Rebases the working copy, made on `base`, onto the tip of `onto`, if + /// that merges cleanly. + fn rebase_onto_tip( + &mut self, + working_copy: &WorkingCopy, + base: TreeId, + onto: &TargetName, + ) -> Result<(), WorkspaceError> { + let tip = self + .store + .head(onto)? + .ok_or_else(|| WorkspaceError::UnknownTarget(onto.clone()))?; + let state = self.store.get(tip.acceptance)?.body().state; + let current = &working_copy.revision; + let merge = self.store.merge(Some(base), current.snapshot, state)?; + if !merge.conflicts.is_empty() { + println!( + "The working copy stays where it is: it conflicts with `{onto}` in {}", + Self::list_paths(&merge.conflicts.paths()) + ); + return Ok(()); + } + let revision = Revision { + predecessors: vec![working_copy.id], + base: Base::Acceptance(tip.acceptance), + snapshot: merge.tree, + author: self.config.user(), + ..current.clone() + }; + let encoded = revision.encode(); + self.write(&[merge.trees, vec![encoded.clone()]])?; + self.checkout(current.snapshot, revision.snapshot)?; + self.point_at(encoded.id().into())?; + println!("Working copy: {}", self.headline(&revision)?); + println!("On top of: {}", self.describe_base(revision.base)?); + Ok(()) + } + + fn print_landing(&self, landing: &Landing, onto: &TargetName) -> Result<(), WorkspaceError> { + match landing { + Landing::Landed { + proposal, + source, + acceptance, + } => { + let landed = match source { + Source::Revision(revision) => format!(", revision {},", revision.short()), + Source::Target(_) => String::new(), + }; + println!( + "Landed {proposal}{landed} on {}", + self.describe_acceptance(acceptance.acceptance)? + ); + } + Landing::Unchanged(proposal) => { + println!("Landing {proposal} would change nothing on `{onto}`"); + } + Landing::Pending(pending) => Self::print_pending(pending), + } + Ok(()) + } + + fn print_pending(pending: &Pending) { + let Pending { + onto, + proposal, + subject, + blocked, + } = pending; + let retry = match proposal { + Proposal::Change(change) => format!("origin land {} --onto {onto}", change.short()), + Proposal::Target(target) => format!("origin land --from {target} --onto {onto}"), + }; + match blocked { + Blocked::Evidence(missing) => { + let missing: Vec<_> = missing.iter().map(ToString::to_string).collect(); + println!( + "The landing of {proposal} on `{onto}` waits for {}", + missing.join(", ") + ); + println!( + " about {}: once they're issued, run `{retry}`", + subject.describe() + ); + } + Blocked::Conflict { paths, .. } => { + println!( + "The landing of {proposal} on `{onto}` conflicts in {}", + Self::list_paths(paths) + ); + println!(" resolve it in a new revision, then run `{retry}`"); + } + } + } +} diff --git a/apps/origin-src/src/workspace/mod.rs b/apps/origin-src/src/workspace/mod.rs index ebe67256..0c123198 100644 --- a/apps/origin-src/src/workspace/mod.rs +++ b/apps/origin-src/src/workspace/mod.rs @@ -14,9 +14,12 @@ //! the files changed, they become a new revision of the change, which //! evolves from the previous one. Records are written before anything //! refers to them: the snapshot's objects, then the revision, then the -//! `workspace` file naming it. +//! `workspace` file naming it. Only the statements a new revision needs, +//! such as its unresolved conflicts, come before it, so it's never found +//! without them. mod annotate; +mod land; pub use annotate::{Annotated, Payload}; @@ -26,15 +29,19 @@ use std::path::{self, Path, PathBuf}; use origin_platform::Native; use origin_shared::{ - AcceptanceBody, AcceptanceId, Base, BlobId, ChangeId, EncodedObject, EntryKind, EntryName, - Kind, NodePublicKey, Principal, Record, RecordId, Revision, RevisionId, Source, Statement, - StatementId, TargetId, TargetName, Tree, TreeEntry, TreeId, + AcceptanceId, Base, BlobId, ChangeId, EncodedObject, Kind, NodePublicKey, Principal, Record, + RecordId, RepoPath, Revision, RevisionId, Source, Statement, StatementId, TargetId, TargetName, + Tree, TreeId, }; -use crate::annotations::AnnotateError; +use crate::ancestry::Ancestry; +use crate::annotations::{AnnotateError, Annotations}; use crate::config::{ConfigError, UserConfig}; +use crate::conflict::Conflicts; use crate::index::{FindChangeError, FindError, Index}; use crate::output::FileDiff; +use crate::policy::PolicyError; +use crate::queue::{LandError, Queue}; use crate::store::{Store, StoreError}; use crate::worktree::{SnapshotError, StatCache, Worktree}; @@ -58,6 +65,8 @@ struct WorkingCopy { /// The revision's record, if recording the draft made a new revision, /// which is not written yet. unsaved: Option, + /// The unresolved conflicts of the revision that the files still have. + conflicts: Conflicts, } impl WorkingCopy { @@ -94,32 +103,10 @@ impl Workspace { let store = Store::new(meta, Native::default()); store.init()?; - let (policy, policy_tree) = Self::policy(&config, &key.public()); + let policy = Self::policy(&config, &key.public()); + let main = Queue::new(&store, &key, config.node().clone()).init(&policy)?; let empty_tree = Tree::default().encode(); let empty_description = EncodedObject::blob(b""); - let genesis = |target: &TargetName, state: &EncodedObject, policy| AcceptanceBody { - target: target.id(), - seq: 0, - prev: None, - state: state.id().into(), - landed: Vec::new(), - evidence: Vec::new(), - policy, - }; - store.write_all(&[ - policy, - policy_tree.clone(), - empty_tree.clone(), - empty_description.clone(), - ])?; - let policy = store.append( - &TargetName::POLICY, - &genesis(&TargetName::POLICY, &policy_tree, None).sign(&key), - )?; - let main = store.append( - &TargetName::MAIN, - &genesis(&TargetName::MAIN, &empty_tree, Some(policy)).sign(&key), - )?; let revision = Revision { change: ChangeId::random().map_err(WorkspaceError::Random)?, @@ -137,7 +124,10 @@ impl Workspace { config, revision: encoded.id().into(), }; - workspace.save(&[vec![encoded]], workspace.revision)?; + workspace.save( + &[vec![empty_description], vec![encoded]], + workspace.revision, + )?; println!( "Initialized a repository in {}: node:{} is the authority of `policy` and `main`", workspace.root.display(), @@ -147,34 +137,30 @@ impl Workspace { Ok(workspace) } - /// The first `policy.toml`, and the snapshot holding it: this node is - /// the authority of `policy` and `main`, and neither requires anything, - /// so a solo project lands instantly. - fn policy(config: &UserConfig, key: &NodePublicKey) -> (EncodedObject, EncodedObject) { + /// The first `policy.toml`: this node is the authority of `policy` and + /// `main`, and neither requires anything, so a solo project lands + /// instantly. + fn policy(config: &UserConfig, key: &NodePublicKey) -> String { let node = config.node(); - let policy = format!( + format!( "# Who may land what. Changing it is a landing on `policy`.\n\ \n\ + # Each node's public key.\n\ [nodes]\n\ \"{node}\" = \"{key}\"\n\ \n\ + # Groups of users and nodes, for requirements:\n\ + # [groups]\n\ + # runners = [\"node:ci-1\"]\n\ + \n\ + # The node ordering each target, and the statements landing needs.\n\ [targets.policy]\n\ authority = \"node:{node}\"\n\ \n\ [targets.main]\n\ - authority = \"node:{node}\"\n" - ); - let policy = EncodedObject::blob(policy.as_bytes()); - let tree: Tree = [( - EntryName::new("policy.toml").expect("a valid name"), - TreeEntry { - kind: EntryKind::File, - id: policy.id(), - }, - )] - .into_iter() - .collect(); - (policy, tree.encode()) + authority = \"node:{node}\"\n\ + # require = [{{ kind = \"ci.test\", verdict = \"pass\", issuer = \"group:runners\" }}]\n" + ) } /// Opens the working copy containing `start`. @@ -225,11 +211,15 @@ impl Workspace { .save(&path) .map_err(|source| WorkspaceError::io(&path, source))?; } + let conflicts = self + .conflicts(self.revision, &revision)? + .remaining(&self.store, files)?; Ok(WorkingCopy { id: self.revision, revision, files, unsaved: None, + conflicts, }) } @@ -247,6 +237,10 @@ impl Workspace { ..working_copy.revision }; let encoded = revision.encode(); + if !working_copy.conflicts.is_empty() { + // Now, so the revision never exists without them. + self.write(&self.revision_layers(Vec::new(), &encoded, &working_copy.conflicts))?; + } Ok(WorkingCopy { id: encoded.id().into(), revision, @@ -255,6 +249,46 @@ impl Workspace { }) } + /// The unresolved conflicts of `revision`, `id`. + fn conflicts(&self, id: RevisionId, revision: &Revision) -> Result { + let annotations = Annotations::open(&self.store)?; + let claims = annotations.claims(RecordId::Revision(id))?; + Ok(Conflicts::of(&self.store, &claims, revision)?) + } + + /// The unresolved conflicts a new change on `base` starts with, from + /// the files of the revision it's on, if it's on one. + fn base_conflicts(&self, base: Base) -> Result { + match base { + Base::Revision(id) => self.conflicts(id, &self.store.get(id)?), + Base::Acceptance(_) => Ok(Conflicts::default()), + } + } + + /// The layers that write `revision`, a new one: `first`, objects it + /// refers to, then the statements published with it, then the revision + /// itself. The statements are a `core.conflict` about it listing + /// `conflicts`, its unresolved ones, if it has any (see + /// [`conflict`](crate::conflict)), and any a caller adds to the second + /// layer. Whoever finds the revision finds them: an error or a crash + /// never publishes it without them, as one batch could, publishing some + /// of its files. + fn revision_layers( + &self, + mut first: Vec, + revision: &EncodedObject, + conflicts: &Conflicts, + ) -> Vec> { + let mut statements = Vec::new(); + if !conflicts.is_empty() { + let subject = RecordId::Revision(revision.id().into()); + let (payload, statement) = conflicts.statement(subject, self.config.user(), None); + first.push(payload); + statements.push(statement.encode()); + } + vec![first, statements, vec![revision.clone()]] + } + /// Snapshots the working copy and saves its draft as a new revision if /// the files changed, for the commands that build on it. fn refresh(&mut self) -> Result { @@ -321,6 +355,9 @@ impl Workspace { ); println!("On top of: {}", self.describe_base(revision.base)?); Self::print_divergence(&index, revision.change); + if !working_copy.conflicts.is_empty() { + Self::print_conflicts(&working_copy.conflicts); + } let changes = self .store .diff(self.base_tree(revision.base)?, working_copy.files)?; @@ -434,7 +471,9 @@ impl Workspace { let id = encoded.id().into(); let recorded = working_copy.unsaved.is_some(); let first = working_copy.unsaved.into_iter().chain([description]); - self.save(&[first.collect(), vec![encoded]], id)?; + // It starts with the files of the one below, and their conflicts. + let layers = self.revision_layers(first.collect(), &encoded, &working_copy.conflicts); + self.save(&layers, id)?; // Edits recorded on a revision that is no longer its change's head // make that change diverge. if recorded { @@ -464,11 +503,43 @@ impl Workspace { let encoded = revision.encode(); let id = encoded.id().into(); let first = working_copy.unsaved.into_iter().chain([description]); - self.save(&[first.collect(), vec![encoded]], id)?; + let layers = self.revision_layers(first.collect(), &encoded, &working_copy.conflicts); + self.save(&layers, id)?; println!("Working copy: {}", self.headline(&revision)?); Ok(()) } + /// `origin resolve ...`: accepts the files at `paths` as they + /// are, resolving their conflicts, with a new revision of the change + /// that keeps the others. Paths without one are skipped. + /// + /// # Errors + /// + /// Fails if the working copy cannot be snapshotted or the revision + /// cannot be written. + pub fn resolve(&mut self, paths: &[RepoPath]) -> Result<(), WorkspaceError> { + let mut working_copy = self.refresh()?; + let resolved = working_copy.conflicts.resolve(paths); + if resolved.is_empty() { + println!("No conflict to resolve there."); + return Ok(()); + } + let revision = Revision { + predecessors: vec![working_copy.id], + author: self.config.user(), + ..working_copy.revision + }; + let encoded = revision.encode(); + let id = encoded.id().into(); + let layers = self.revision_layers(Vec::new(), &encoded, &working_copy.conflicts); + self.save(&layers, id)?; + println!("Resolved {}", Self::list_paths(&resolved)); + if !working_copy.conflicts.is_empty() { + Self::print_conflicts(&working_copy.conflicts); + } + Ok(()) + } + /// `origin edit `: switches the working copy to the change's /// head. /// @@ -546,7 +617,11 @@ impl Workspace { author: self.config.user(), }; let encoded = revision.encode(); - self.write(&[vec![reason, empty], vec![statement, encoded.clone()]])?; + let conflicts = self.base_conflicts(revision.base)?; + let mut layers = self.revision_layers(vec![reason, empty], &encoded, &conflicts); + // Published before the revision that replaces the change. + layers[1].push(statement); + self.write(&layers)?; self.checkout(working_copy.revision.snapshot, revision.snapshot)?; self.point_at(encoded.id().into())?; println!("Abandoned change {}", change.short()); @@ -556,9 +631,14 @@ impl Workspace { } /// `origin converge `: ends a change's divergence with a - /// revision evolving from all its heads. It keeps the files and - /// description of the working copy, if it is one of the heads, or else - /// of the newest head. + /// revision evolving from all its heads. It merges their files, file by + /// file, with the changes each made since the latest revision they all + /// evolve from. It keeps the base and the description of the working + /// copy, if it is one of the heads, or else of the newest head. Where + /// heads changed the same text file, it holds each version between + /// conflict markers, and a `core.conflict` statement lists the files, + /// so the revision doesn't land until a snapshot without the markers: + /// see [`conflict`](crate::conflict). /// /// # Errors /// @@ -577,28 +657,87 @@ impl Workspace { } else { heads[heads.len() - 1] }; + let kept_revision: Revision = self.store.get(kept)?; + let base = self.merge_base(&heads)?; + let mut snapshot = kept_revision.snapshot; + let mut merged = Conflicts::default(); + for (merged_heads, &head) in heads.iter().filter(|&&head| head != kept).enumerate() { + // Past the first merge, our side holds every head merged so far. + let ours = if merged_heads == 0 { + kept.short() + } else { + "merged heads".to_owned() + }; + let merge = self.store.merge_marked( + base, + snapshot, + self.store.get(head)?.snapshot, + [&ours, &head.short()], + )?; + // The next merge reads this one's trees. + self.write(&[merge.files.into_iter().chain(merge.trees).collect()])?; + snapshot = merge.tree; + merged = merged.with(merge.conflicts); + } + // The heads' own, where the merge kept their side. + let mut conflicts = Conflicts::default(); + for &head in &heads { + let unresolved = self.conflicts(head, &self.store.get(head)?)?; + conflicts = conflicts.with(unresolved.remaining(&self.store, snapshot)?); + } + let conflicts = conflicts.with(merged); let revision = Revision { predecessors: heads.clone(), + snapshot, author: self.config.user(), - ..self.store.get(kept)? + ..kept_revision }; let encoded = revision.encode(); let id = encoded.id().into(); - self.write(&[vec![encoded]])?; + self.write(&self.revision_layers(Vec::new(), &encoded, &conflicts))?; if change == working_copy.revision.change { self.checkout(working_copy.revision.snapshot, revision.snapshot)?; self.point_at(id)?; } println!( - "Converged change {}: revision {} evolves from {}, with the files of {}", + "Converged change {}: revision {} evolves from {}, merging their files", change.short(), id.short(), Self::list(&heads), - kept.short() ); + if !conflicts.is_empty() { + Self::print_conflicts(&conflicts); + } Ok(()) } + /// Lists a revision's unresolved conflicts, and how to resolve them. + fn print_conflicts(conflicts: &Conflicts) { + println!( + "Conflicts: {} hold both versions between conflict markers, or one side for a \ + binary file. Edit them, or run `origin resolve ...` to accept them as they \ + are.", + Self::list_paths(&conflicts.paths()) + ); + } + + /// `paths`, separated by commas. + fn list_paths(paths: &[RepoPath]) -> String { + let paths: Vec<_> = paths.iter().map(ToString::to_string).collect(); + paths.join(", ") + } + + /// The snapshot to merge `heads`, revisions of one change, from: that of + /// the newest revision they all evolve from, or the + /// [merge](Ancestry::merge_base) of several, if each lacks another. + fn merge_base(&self, heads: &[RevisionId]) -> Result, WorkspaceError> { + let ancestry = Ancestry::walk(heads, |id| { + let revision: Revision = self.store.get(id)?; + Ok::<_, StoreError>((revision.predecessors, revision.snapshot)) + })?; + Ok(ancestry.merge_base(&self.store, heads)?) + } + /// `origin log []`: prints the working copy's stack of changes /// down to the target it's on, then the target's acceptances, newest /// first. With a target, prints that target's acceptances instead, @@ -951,6 +1090,12 @@ pub enum WorkspaceError { /// A statement cannot be issued. #[error(transparent)] Annotate(#[from] AnnotateError), + /// The policy cannot be read. + #[error(transparent)] + Policy(#[from] PolicyError), + /// A landing failed. + #[error(transparent)] + Land(#[from] LandError), } impl WorkspaceError { diff --git a/apps/origin-src/tests/cli.rs b/apps/origin-src/tests/cli.rs index 219b9b43..34fe0e8f 100644 --- a/apps/origin-src/tests/cli.rs +++ b/apps/origin-src/tests/cli.rs @@ -260,19 +260,118 @@ fn divergence_shows_until_converged() { converged.starts_with(&format!("Converged change {change}")), "{converged}" ); - assert_eq!( - repo.read("file"), - "right\n", - "the working copy's files are kept" + assert!( + converged.contains("Conflicts: file hold both versions"), + "{converged}" ); - assert!(!repo.ok(&["status"]).contains("divergent")); + // Both versions, the working copy's first, between markers. + let file = repo.read("file"); + assert!(file.starts_with("<<<<<<< "), "{file}"); + assert!(file.contains("\nright\n=======\nleft\n>>>>>>> "), "{file}"); + let status = repo.ok(&["status"]); + assert!(!status.contains("divergent"), "{status}"); + assert!(status.contains("Conflicts: file hold"), "{status}"); assert!(repo.ok(&["evolog"]).contains(" converges ")); + + // It doesn't land until the file changes: describing it keeps the + // conflict, an edit resolves it, whatever the file then holds. + repo.ok(&["describe", "Shared work, converged"]); + let refused = repo.fails(&["land"]); + assert!(refused.contains("unresolved conflicts"), "{refused}"); + assert!(refused.contains("edit file"), "{refused}"); + repo.write("file", "Heading\n=======\nboth\n"); + assert!(!repo.ok(&["status"]).contains("Conflicts")); + assert!(repo.ok(&["land"]).starts_with("Landed change")); assert!( repo.fails(&["converge", &change]) .contains("is not divergent") ); } +#[test] +fn binary_conflicts_resolve_only_when_accepted() { + let repo = Repo::new(); + repo.write("image", "base\0"); + repo.write("text", "base\n"); + repo.ok(&["describe", "Shared work"]); + let change = repo.change(); + let pointer = repo.path(".origin/workspace"); + let started = fs::read_to_string(&pointer).unwrap(); + repo.write("image", "left\0"); + repo.write("text", "left\n"); + repo.ok(&["describe", "Shared work"]); + fs::write(&pointer, started).unwrap(); + repo.write("image", "right\0"); + repo.write("text", "right\n"); + repo.ok(&["converge", &change]); + assert_eq!(repo.read("image"), "right\0", "the working copy's side"); + + // An edit elsewhere leaves it unresolved, even without markers. + repo.write("unrelated", "docs\n"); + assert!(repo.fails(&["land"]).contains("unresolved conflicts")); + let resolved = repo.ok(&["resolve", "image", "unrelated"]); + assert!(resolved.starts_with("Resolved image\n"), "{resolved}"); + assert!(resolved.contains("Conflicts: text hold"), "{resolved}"); + let status = repo.ok(&["status"]); + assert!(status.contains("\nConflicts: text hold"), "{status}"); + repo.ok(&["resolve", "text"]); + assert!(repo.ok(&["land"]).starts_with("Landed change")); +} + +#[test] +fn a_revision_is_published_only_after_its_conflicts() { + use origin_shared::{EncodedObject, Record, RecordId, Revision, RevisionId}; + use origin_src::{annotations::Annotations, conflict::Conflicts, store::Store}; + + let repo = Repo::new(); + repo.write("file", "base\n"); + repo.ok(&["describe", "Shared work"]); + let change = repo.change(); + let pointer = repo.path(".origin/workspace"); + let started = fs::read_to_string(&pointer).unwrap(); + repo.write("file", "left\n"); + repo.ok(&["describe", "Shared work"]); + fs::write(&pointer, started).unwrap(); + repo.write("file", "right\n"); + repo.ok(&["converge", &change]); + + // The `core.conflict` statement `describe` issues about its revision. + let store = Store::new(repo.path(".origin"), origin_platform::Native::default()); + let converged: RevisionId = fs::read_to_string(&pointer) + .unwrap() + .trim() + .parse() + .unwrap(); + let revision: Revision = store.get(converged).unwrap(); + let claims = Annotations::open(&store) + .unwrap() + .claims(RecordId::Revision(converged)) + .unwrap(); + let conflicts = Conflicts::of(&store, &claims, &revision).unwrap(); + let issuer = revision.author.clone(); + let described = Revision { + predecessors: vec![converged], + description: EncodedObject::blob(b"Described").id().into(), + ..revision + } + .encode(); + let subject = RecordId::Revision(described.id().into()); + let (_, statement) = conflicts.statement(subject, issuer, None); + let hex = statement.encode().id().to_hex(); + + // Writing it fails: its directory is read-only. + let dir = repo.path(&format!(".origin/objects/loose/{}", &hex[..2])); + fs::create_dir_all(&dir).unwrap(); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).unwrap(); + repo.fails(&["describe", "Described"]); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).unwrap(); + + // So the revision, which would hide the conflict, wasn't written. + assert!(!store.contains(described.id()).unwrap()); + let refused = repo.fails(&["land"]); + assert!(refused.contains("unresolved conflicts"), "{refused}"); +} + #[test] fn abandoning_the_current_change_restores_its_base() { let repo = Repo::new(); @@ -823,3 +922,272 @@ fn statements_are_promoted_where_their_inputs_are_unchanged() { ); assert!(repo.fails(&["annotate", "@", "ci.lint"]).contains("--pass")); } + +#[test] +fn landing_moves_the_working_copy_onto_the_target() { + let repo = Repo::new(); + repo.write("a", "a\n"); + repo.ok(&["describe", "Add a"]); + let first = repo.change(); + repo.ok(&["new", "Add b"]); + let second = repo.change(); + repo.write("b", "b\n"); + repo.ok(&["new"]); + let empty = repo.change(); + + // The stack lands from the bottom; the empty working copy has nothing + // to land, and moves onto the new tip. + let land = repo.ok(&["land"]); + let lines: Vec<_> = land.lines().collect(); + assert!( + lines[0].starts_with(&format!("Landed change {first}, revision ")), + "{land}" + ); + assert!(lines[0].ends_with(", on main #1"), "{land}"); + assert!( + lines[1].starts_with(&format!("Landed change {second}, ")), + "{land}" + ); + assert_eq!( + lines[2], + format!("Landing change {empty} would change nothing on `main`") + ); + assert!( + repo.ok(&["status"]) + .contains("On top of: main #2\nNo changes.\n") + ); + + // Landing the working copy's change starts a new one on the tip. + repo.write("c", "c\n"); + let land = repo.ok(&["land"]); + assert!(land.contains(" on main #3\nWorking copy: "), "{land}"); + assert_ne!(repo.change(), empty); + assert!( + repo.ok(&["status"]) + .ends_with("On top of: main #3\nNo changes.\n") + ); + assert_eq!(repo.read("c"), "c\n"); + + let targets = repo.ok(&["targets"]); + assert!(targets.starts_with("main #3 "), "{targets}"); + assert!(targets.contains("(node:ada)\npolicy #0 "), "{targets}"); +} + +#[test] +fn a_change_the_target_already_has_moves_onto_its_tip() { + let repo = Repo::new(); + repo.write("f", "f\n"); + let ours = repo.change(); + repo.ok(&["new", "--on", "main", "The same file"]); + repo.write("f", "f\n"); + repo.ok(&["land"]); + repo.ok(&["edit", &ours]); + let land = repo.ok(&["land"]); + assert!(land.contains(" would change nothing on `main`\n"), "{land}"); + + // Deleting the file is then a change to what `main` has. + fs::remove_file(repo.path("f")).unwrap(); + let land = repo.ok(&["land"]); + assert!(land.starts_with("Landed change "), "{land}"); + assert!(!repo.path("f").exists()); +} + +#[test] +fn targets_fork_and_land_on_each_other() { + let repo = Repo::new(); + repo.ok(&["target", "new", "overhaul", "--from", "main"]); + assert!( + repo.fails(&["target", "new", "overhaul", "--from", "main"]) + .contains("already named `overhaul`") + ); + repo.ok(&["new", "--on", "overhaul", "Rework"]); + repo.write("rework", "new\n"); + let land = repo.ok(&["land"]); + assert!( + land.contains(" on overhaul #1\n"), + "lands where it's made: {land}" + ); + + let land = repo.ok(&["land", "--from", "overhaul", "--onto", "main"]); + assert_eq!(land, "Landed `overhaul` on main #1\n"); + assert!(repo.ok(&["log", "main"]).starts_with("◆ main #1 ")); + assert!(repo.ok(&["log", "main"]).contains("\n overhaul #1\n")); +} + +/// Lands a policy that makes `main` require `ci.test` to pass, from the +/// runners, ada among them. +fn require_tests(repo: &Repo) { + repo.ok(&["new", "--on", "policy", "Require tests"]); + let policy = repo + .read("policy.toml") + .replace("# [groups]", "[groups]") + .replace( + "# runners = [\"node:ci-1\"]", + "runners = [\"node:ci-1\", \"user:ada\"]", + ) + .replace("# require", "require"); + repo.write("policy.toml", &policy); + assert!(repo.ok(&["land"]).contains(" on policy #1\n")); +} + +#[test] +fn results_there_count_over_promoted_ones() { + let repo = Repo::new(); + require_tests(&repo); + repo.ok(&["new", "--on", "main", "Library"]); + repo.write("src/lib.rs", "fn a() {}\n"); + repo.ok(&["annotate", "@", "ci.test", "--pass", "--inputs", "src"]); + assert!(repo.ok(&["land"]).contains(" on main #1\n")); + // Each change below keeps `src`, so the pass is promoted to it. + let promoted = |description: &str| { + repo.ok(&["new", "--on", "main", description]); + repo.write(&format!("{description}.md"), "docs\n"); + assert!(repo.ok(&["promote"]).contains("Promoted ci.test pass")); + }; + let waits = || { + let landing = repo.ok(&["land"]); + assert!(landing.starts_with("The landing of change "), "{landing}"); + }; + // A pass there, then taken back, takes the promoted one with it. + promoted("withdrawn"); + repo.ok(&["annotate", "@", "ci.test", "--pass", "--inputs", "src"]); + repo.ok(&["annotate", "@", "ci.test", "--withdraw"]); + waits(); + // A failure there wins over the promoted pass. + promoted("failed"); + repo.ok(&["annotate", "@", "ci.test", "--fail", "--inputs", "src"]); + waits(); +} + +#[test] +fn landings_wait_for_the_evidence_policy_requires() { + let repo = Repo::new(); + require_tests(&repo); + assert!( + repo.ok(&["targets"]) + .contains(" requires ci.test pass from group:runners\n") + ); + + repo.ok(&["new", "--on", "main", "Feature"]); + let change = repo.change(); + repo.write("src/lib.rs", "fn a() {}\n"); + let waiting = repo.ok(&["land"]); + assert!( + waiting.starts_with(&format!( + "The landing of change {change} on `main` waits for ci.test pass from group:runners\n" + )), + "{waiting}" + ); + assert_eq!(repo.ok(&["land", "--status"]), waiting); + // A result withdrawn is no evidence. + repo.ok(&["annotate", "@", "ci.test", "--pass", "--inputs", "src"]); + repo.ok(&["annotate", "@", "ci.test", "--withdraw"]); + assert_eq!(repo.ok(&["land"]), waiting); + repo.ok(&["annotate", "@", "ci.test", "--pass", "--inputs", "src"]); + assert!(repo.ok(&["land"]).contains(" on main #1\n")); + assert_eq!(repo.ok(&["land", "--status"]), "No landing waits.\n"); + + // Only the authority lands: here, after a policy edit hands `main` to + // another node. + repo.ok(&["new", "--on", "policy"]); + let policy = repo.read("policy.toml").replace( + "[targets.main]\nauthority = \"node:ada\"", + "[targets.main]\nauthority = \"node:hosted\"", + ); + // Another node's key: any valid one will do. + let key = policy + .lines() + .find_map(|line| line.strip_prefix("\"ada\" = ")) + .unwrap(); + repo.write( + "policy.toml", + &policy.replace("[nodes]\n", &format!("[nodes]\nhosted = {key}\n")), + ); + repo.ok(&["land"]); + repo.ok(&["new", "--on", "main"]); + repo.write("x", "x\n"); + assert!( + repo.fails(&["land"]) + .contains("node:hosted is the authority of `main`") + ); +} + +#[test] +fn conflicting_landings_are_flagged_and_wait() { + let repo = Repo::new(); + repo.ok(&["describe", "One"]); + repo.write("shared", "one\n"); + let one = repo.change(); + repo.ok(&["new", "--on", "main", "Two"]); + repo.write("shared", "two\n"); + let two = repo.change(); + repo.ok(&["land", &one]); + + let conflict = repo.ok(&["land"]); + assert!( + conflict.starts_with(&format!( + "The landing of change {two} on `main` conflicts in shared\n" + )), + "{conflict}" + ); + assert!( + repo.ok(&["statements"]) + .contains(" core.conflict by node:ada\n") + ); + assert_eq!(repo.ok(&["land", "--status"]), conflict); +} + +#[test] +fn converge_merges_the_files_of_every_head() { + let repo = Repo::new(); + for path in ["a", "b", "c"] { + repo.write(path, "base\n"); + } + repo.ok(&["describe", "Shared work"]); + let change = repo.change(); + // Three agents revise the same revision, each changing its own file. + let pointer = repo.path(".origin/workspace"); + let started = fs::read_to_string(&pointer).unwrap(); + for changed in ["a", "b", "c"] { + fs::write(&pointer, &started).unwrap(); + for path in ["a", "b", "c"] { + repo.write( + path, + if path == changed { + "changed\n" + } else { + "base\n" + }, + ); + } + repo.ok(&["describe", "Shared work"]); + } + // Merging a third head reads the tree merging the first two. + let converged = repo.ok(&["converge", &change]); + assert!( + converged.ends_with(", merging their files\n"), + "{converged}" + ); + for path in ["a", "b", "c"] { + assert_eq!(repo.read(path), "changed\n", "{path}"); + } +} + +#[test] +fn converge_labels_the_side_holding_merged_heads() { + let repo = Repo::new(); + repo.write("file", "base\n"); + repo.ok(&["describe", "Shared work"]); + let change = repo.change(); + let pointer = repo.path(".origin/workspace"); + let started = fs::read_to_string(&pointer).unwrap(); + for side in ["one", "two", "three"] { + fs::write(&pointer, &started).unwrap(); + repo.write("file", &format!("{side}\n")); + repo.ok(&["describe", "Shared work"]); + } + repo.ok(&["converge", &change]); + // The third head merges into both versions of the first two. + let file = repo.read("file"); + assert!(file.starts_with("<<<<<<< merged heads\n<<<<<<< "), "{file}"); +} diff --git a/crates/origin-shared/src/object/acceptance.rs b/crates/origin-shared/src/object/acceptance.rs index 95ef6464..ecc82997 100644 --- a/crates/origin-shared/src/object/acceptance.rs +++ b/crates/origin-shared/src/object/acceptance.rs @@ -1,10 +1,11 @@ use std::fmt; +use std::str::FromStr; use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; use super::{DecodeError, Field, ObjectKind, Reader, Record, fields, tagged}; use crate::id::hex; -use crate::{AcceptanceId, RevisionId, StatementId, TargetId, TreeId}; +use crate::{AcceptanceId, ParseIdError, RevisionId, StatementId, TargetId, TreeId}; pub use ed25519_dalek::Signature; @@ -178,6 +179,21 @@ impl fmt::Display for NodePublicKey { } } +/// Parses the hex [`NodePublicKey`] displays as. +impl FromStr for NodePublicKey { + type Err = ParseIdError; + + fn from_str(input: &str) -> Result { + hex::decode(input) + .and_then(|bytes| VerifyingKey::from_bytes(&bytes).ok()) + .map(Self) + .ok_or_else(|| ParseIdError { + what: "node public key", + input: input.to_owned(), + }) + } +} + /// A signature that the claimed signer did not make. #[derive(Debug, thiserror::Error)] #[error("the signature does not match the signer's key")] diff --git a/crates/origin-shared/src/object/mod.rs b/crates/origin-shared/src/object/mod.rs index fe3742b3..4567e347 100644 --- a/crates/origin-shared/src/object/mod.rs +++ b/crates/origin-shared/src/object/mod.rs @@ -396,6 +396,9 @@ pub(crate) mod tests { acceptance.verify(&sample_key().public()).unwrap(); let other = NodeKey::from_bytes(&[1; 32]); assert!(acceptance.verify(&other.public()).is_err()); + let public = sample_key().public(); + assert_eq!(public.to_string().parse::().unwrap(), public); + assert!("ab".parse::().is_err()); // Any change to the body breaks the signature: here the sequence // number, right after the kind and the target id. diff --git a/crates/origin-shared/src/object/statement.rs b/crates/origin-shared/src/object/statement.rs index 616abbe2..aa171e9c 100644 --- a/crates/origin-shared/src/object/statement.rs +++ b/crates/origin-shared/src/object/statement.rs @@ -1,11 +1,13 @@ use std::borrow::Cow; use std::collections::HashSet; use std::fmt; +use std::str::FromStr; use super::{DecodeError, Field, ObjectKind, Reader, Record, fields, tagged}; use crate::name::names; use crate::{ - AcceptanceId, BlobId, Digest, Principal, RepoPath, RevisionId, StatementId, TreeEntry, TreeId, + AcceptanceId, BlobId, Digest, InvalidName, Principal, RepoPath, RevisionId, StatementId, + TreeEntry, TreeId, }; /// An attributable claim about another record: a check result, a review, a @@ -210,6 +212,23 @@ impl fmt::Display for Verdict { } } +/// Parses the names [`Verdict`] displays as: `pass`, `fail` and `neutral`. +impl FromStr for Verdict { + type Err = InvalidName; + + fn from_str(input: &str) -> Result { + match input { + "pass" => Ok(Self::Pass), + "fail" => Ok(Self::Fail), + "neutral" => Ok(Self::Neutral), + _ => Err(InvalidName { + what: "verdict", + name: input.to_owned(), + }), + } + } +} + impl Field for Verdict { fn write(&self, out: &mut Vec) { out.push(*self as u8); @@ -402,4 +421,12 @@ mod tests { } assert!(!Kind::new("corex.test").unwrap().is_reserved()); } + + #[test] + fn verdicts_parse_as_they_display() { + for verdict in [Verdict::Pass, Verdict::Fail, Verdict::Neutral] { + assert_eq!(verdict.to_string().parse::().unwrap(), verdict); + } + assert!("Pass".parse::().is_err()); + } } diff --git a/docs/design.md b/docs/design.md index d8958778..5eb3d4f2 100644 --- a/docs/design.md +++ b/docs/design.md @@ -379,7 +379,11 @@ and stops at the first one that waits. several, as git's recursive strategy does. The merge is checked as a revision on the tip, authored by the queue's node, so results about the same merge on another target never count. - - A landing that changes nothing is skipped. + - A landing that changes nothing is skipped: the target already has it. + But a merge of another target's tip that changes nothing still lands, + as an acceptance of the same state that needs no evidence, so later + merges start from it, unless either tip is already in the other's + history: then neither has anything the other lacks. 3. **On conflict,** issue a `core.conflict` statement and keep the landing blocked. Nothing fails, and landing again retries it. 4. **Check the policy** in force *now*: