From e92d42edda110f430b552249d32bbd2671d5c569 Mon Sep 17 00:00:00 2001 From: piquel Date: Sat, 17 Oct 2026 18:39:13 +0200 Subject: [PATCH] feat: add clone, push, fetch and remote landing Nodes share work with one remote: a server over HTTP, or another repository's path served by an in-process OriginService. `Remote` answers the same calls either way. A clone follows `policy` and one target: it fetches their chains, with the revisions they landed and the records of their changes, and only the tip's snapshot. Each acceptance must follow the one before it, and is verified against the authority of the policy governing it: an acceptance of `policy` by the one before it, any other by the policy it names, which must be on the verified chain of `policy`. Acceptances only stored, like a revision's base, govern nothing. Other snapshots are fetched when a command reads them, marked while they're fetched, so an interrupted fetch is never taken for a whole snapshot. Snapshots only leave a repository by walking them (`POST snapshots/fetch`, skipping what the caller's snapshots hold at the same path, optionally under some paths), so #6 can filter reads by path; records are fetched by id, with the description or payload they carry. A move of the working copy fetches what it needs before it publishes `moving`, so one that cannot reach the remote moves nothing. Fetched chains append several acceptances at once: recovering such an append a crash cut short logs that the target advanced to each of them. `push` compares each change's heads with the remote's inventory and refuses a stale change before uploading anything, and the server checks the heads again as it publishes the revisions (`PUT records?expect=`), so a push racing another is refused too. It sends snapshots, then revisions, then statements, each after what it needs, and says which statements stay because they need one the remote lacks. It only shares revisions based on acceptances the remote has. Transfers go in parts of at most 64 MiB, both ways. `fetch` brings new acceptances, then the records of the changes the remote's events since the last fetch say moved, so a fetch that brings nothing doesn't grow with the history; without a cursor, or once it expired, it compares every change it knows. `pull` fetches, then rebases the working copy onto its target's tip. `land`, `land --from` and `target new` go to the remote's queue for every target it has; `watch` long-polls its events. --- AGENTS.md | 11 +- README.md | 42 +- apps/origin-src/Cargo.toml | 6 +- apps/origin-src/benches/sync.rs | 328 +++ apps/origin-src/src/cli.rs | 61 +- apps/origin-src/src/events.rs | 7 +- apps/origin-src/src/http.rs | 117 +- apps/origin-src/src/index.rs | 5 + apps/origin-src/src/lib.rs | 2 + apps/origin-src/src/oplog.rs | 50 +- apps/origin-src/src/queue.rs | 24 +- apps/origin-src/src/remote.rs | 245 +++ apps/origin-src/src/server.rs | 6 +- apps/origin-src/src/service.rs | 369 +++- apps/origin-src/src/store/mod.rs | 126 +- apps/origin-src/src/sync.rs | 2296 +++++++++++++++++++++ apps/origin-src/src/targets.rs | 305 ++- apps/origin-src/src/tree.rs | 253 ++- apps/origin-src/src/workspace/land.rs | 115 +- apps/origin-src/src/workspace/mod.rs | 146 +- apps/origin-src/src/workspace/ops.rs | 20 +- apps/origin-src/src/workspace/registry.rs | 17 +- apps/origin-src/src/workspace/remote.rs | 380 ++++ apps/origin-src/tests/cli.rs | 211 +- apps/origin-src/tests/common/mod.rs | 328 +++ apps/origin-src/tests/http.rs | 148 +- apps/origin-src/tests/serve.rs | 50 +- apps/origin-src/tests/sync.rs | 224 ++ crates/origin-client/src/lib.rs | 47 +- crates/origin-shared/src/api.rs | 150 +- crates/origin-shared/src/event.rs | 4 + crates/origin-shared/src/landing.rs | 12 + crates/origin-shared/src/pack.rs | 130 +- docs/openapi.json | 268 ++- tools/xtask/src/main.rs | 397 +++- 35 files changed, 6276 insertions(+), 624 deletions(-) create mode 100644 apps/origin-src/benches/sync.rs create mode 100644 apps/origin-src/src/remote.rs create mode 100644 apps/origin-src/src/sync.rs create mode 100644 apps/origin-src/src/workspace/remote.rs create mode 100644 apps/origin-src/tests/common/mod.rs create mode 100644 apps/origin-src/tests/sync.rs diff --git a/AGENTS.md b/AGENTS.md index 278bc0eb..99cad280 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,8 @@ review bots read and write at the same time, written in Rust. server. `OriginService` (`service.rs`) is the one API over a repository: the HTTP routes (`http.rs`) and the command line's commands that the API offers only translate into its calls, so neither duplicates the other. + `sync.rs` shares records with a remote (`remote.rs`), which answers the + same calls over the SDK or, for a local path, in-process. - `tools/xtask`: development tasks, run with `cargo xtask`, such as timing origin against git (`just bench-git`). @@ -30,15 +32,18 @@ to a hot path (hashing, storage, snapshots, transfer) should come with a benchmark, and its numbers in the pull request. - `just bench`: the divan micro-benchmarks (encoding, store writes, - snapshots, input digests and promotion, landing, reading events, and the - server's request latency and long-poll wakeups). + snapshots, input digests and promotion, landing, reading events, the + server's request latency and long-poll wakeups, syncing a 20,000-file + tree over HTTP, and a fetch that brings nothing from a long history). - `just bench-git`: times origin against git on a 20,000-file tree with hyperfine, and rewrites `docs/benchmarks.md`. End-to-end tests run the `origin` binary with `ORIGIN_CONFIG_DIR` set to a temporary directory, which holds the user's `config.toml` and node key. The HTTP tests drive every SDK method against a real server. One fails -while `docs/openapi.json` is stale: `just openapi` rewrites it. +while `docs/openapi.json` is stale: `just openapi` rewrites it. The sync +tests clone, push and land through `origin serve` and through local paths; +`tests/common` holds the helpers every end-to-end test shares. ## Commits & Pull Requests diff --git a/README.md b/README.md index af62c6d6..d844cfd5 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,13 @@ origin workspace list | rm origin op log [-n ] # what this node did, operation by operation, in every workspace origin undo # move this workspace's working copy back one operation -origin watch [--kind ] [--subject ] [--since ] # events, as they happen +origin watch [--kind ] [--subject ] [--since ] [--local] # events, as they happen + +origin clone [] [--target ] # follow a remote's target, starting a change on its tip +origin remote [] # show or set the remote: http://host:port/, or a path +origin push [...] # share changes and the ones below them; lands nothing +origin fetch [--target ] [--change ] # new acceptances, and the records of known changes +origin pull # fetch, then move the working copy onto its target's tip ``` Landing rebases a change onto the target's tip, file by file, checks the @@ -81,6 +87,25 @@ op log, one a line starting with a cursor: `statement.issued`, takes an event name or a statement kind (`core.conflict`, `ci.*`), and `--since` resumes after a cursor. +A repository can have one remote, a server or another repository's path, +shared by its workspaces. A clone holds the chains of the targets it +follows, `policy` and one other, verified against their authorities' keys, +and the records of the changes it knows, which each `fetch` brings as they +move: it asks the remote's events what moved since the last fetch, so a +fetch that brings nothing is quick however long the history. It fetches +snapshots only when a command reads them, starting with its target's tip. +`push` shares revisions, their files and statements, and refuses a change +the remote has a newer revision of, even one pushed while it sends: +`fetch`, then `converge` it. A statement that needs one the remote lacks +stays, and the push says so. A landing is decided on the evidence the +remote holds when it's tried: send a statement together with the one it +supersedes, as `push` does, or a landing waiting on the first may take it. +`land`, `land --from` and `target +new` go to the remote's queue for every target the remote has, after pushing +what lands, and `watch` long-polls the remote's events (`--local` for this +node's). Fetching never changes your files; `pull` then moves the working +copy onto its target's new tip, like a landing elsewhere does. + Changes are named by a unique prefix of their id, or `@` for the working copy's. A revision is `@`, a change (for its head), or a unique prefix of the revision's id. Your name and node key live in `$ORIGIN_CONFIG_DIR`, or @@ -119,11 +144,18 @@ key = "node.key" # its secret key, created if missing demo = "repos/demo" ``` -The HTTP API, under `/api/v1`, puts and fetches records in batches, lists -statements, revisions, targets and their logs, lands changes and targets, -and streams events by long-poll: `GET +The HTTP API, under `/api/v1`, puts records in batches, if the changes +they revise are where the client saw them, and fetches them by id, walks +snapshots for the trees and blobs a client lacks (the only way files leave +a server), in parts of at most 64 MiB each way, compares the records of +changes, lists statements, revisions, targets and their logs, lands +changes and targets, and streams +events by long-poll: `GET /api/v1/repos//events?cursor=&wait=30s` answers as soon as there -are events after the cursor, with a JSON batch and the next cursor. +are events after the cursor, with a JSON batch, the next cursor, and the +generation of the op log it's of: pass it back as `generation`, and a +cursor of another log, such as from before the repository was restored, +is refused as expired. [`docs/openapi.json`](docs/openapi.json) describes it, and [`origin-client`](crates/origin-client) is its Rust SDK. Nothing authenticates callers yet. diff --git a/apps/origin-src/Cargo.toml b/apps/origin-src/Cargo.toml index f036de86..c4bc359a 100644 --- a/apps/origin-src/Cargo.toml +++ b/apps/origin-src/Cargo.toml @@ -20,6 +20,7 @@ anyhow.workspace = true axum.workspace = true clap.workspace = true getrandom.workspace = true +origin-client.workspace = true origin-platform.workspace = true origin-shared.workspace = true serde.workspace = true @@ -35,7 +36,6 @@ zstd.workspace = true [dev-dependencies] divan.workspace = true -origin-client.workspace = true # The benchmarks build their working trees with the fixtures. origin-src = { workspace = true, features = ["fixture"] } serde_json.workspace = true @@ -65,6 +65,10 @@ harness = false name = "packs" harness = false +[[bench]] +name = "sync" +harness = false + [[bench]] name = "workspace" harness = false diff --git a/apps/origin-src/benches/sync.rs b/apps/origin-src/benches/sync.rs new file mode 100644 index 00000000..f3a33fd8 --- /dev/null +++ b/apps/origin-src/benches/sync.rs @@ -0,0 +1,328 @@ +//! Sync over HTTP on localhost, through the SDK, on a 20k-file tree: the +//! snapshot a clone fetches, fetching a snapshot with 100 files changed +//! since one the client has, and pushing a revision with 100 files edited. +//! And a fetch that brings nothing, from a repository with a long history, +//! with the remote's event cursor as of the last fetch and without, as +//! every fetch was before cursors. Each includes the server's work and the +//! client's durable writes, so run on a real disk. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, LazyLock}; + +use divan::Bencher; +use origin_platform::Native; +use origin_shared::api::LandRequest; +use origin_shared::landing::{Landing, Proposal}; +use origin_shared::pack::Pack; +use origin_shared::{ + Base, ChangeId, EncodedObject, Kind, NodeKey, Record, RecordId, Revision, Statement, + TargetName, TreeId, Verdict, +}; +use origin_src::fixture::FixtureTree; +use origin_src::http::Api; +use origin_src::policy::Policy; +use origin_src::remote::{Remote, RemoteUrl}; +use origin_src::server::Repositories; +use origin_src::service::{Caller, Node, OriginService}; +use origin_src::store::Store; +use origin_src::sync::{Cursor, Fetched, Sync}; +use origin_src::worktree::{StatCache, Worktree}; +use tokio::runtime::Runtime; + +fn main() { + divan::main(); +} + +const TREE: FixtureTree = FixtureTree::LARGE; + +/// The files each edit changes. +const EDITS: usize = 100; + +/// The runs of the benchmarks that need a snapshot of their own each. +const SAMPLES: u32 = 10; + +/// A server hosting the fixture as `main`'s tip, and snapshots of it with +/// 100 files edited; and a client holding the tip, with snapshots of its +/// own edits. +struct Setup { + _dirs: [tempfile::TempDir; 3], + _server: Runtime, + url: RemoteUrl, + node: Node, + /// The tip of `main`, the whole fixture, and its snapshot. + tip: Base, + snapshot: TreeId, + /// Edits only the server has. + theirs: Vec, + client: Store, + /// Edits only the client has. + ours: Vec, +} + +static SETUP: LazyLock = LazyLock::new(|| { + let [files, server, client] = [(); 3].map(|()| tempfile::tempdir().unwrap()); + TREE.write(files.path()).unwrap(); + let node = Node { + name: "hosted".parse().unwrap(), + key: Arc::new(NodeKey::from_bytes(&[7; 32])), + }; + let policy = Policy::initial(&node.name, &node.key.public()); + let service = OriginService::create(server.path().join("demo"), node.clone(), &policy).unwrap(); + let mut cache = StatCache::default(); + let mut snapshot_into = |store: &Store, version: Option| { + if let Some(version) = version { + TREE.edit(files.path(), EDITS, version).unwrap(); + } + let (tree, new) = Worktree::new(files.path(), store).snapshot(&cache).unwrap(); + cache = new.unwrap_or_default(); + tree + }; + let snapshot = snapshot_into(service.store(), None); + let tip = land(&service, snapshot); + let theirs = (1..=u64::from(SAMPLES)) + .map(|version| snapshot_into(service.store(), Some(version))) + .collect(); + + let (runtime, url) = serve(service); + + let store = Store::new(client.path().join(".origin"), Native::default()); + store.init().unwrap(); + let remote = Remote::open(&url, node.clone()).unwrap(); + let sync = Sync::new(&store, &remote, "laptop".parse().unwrap()); + sync.fetch_chains(&[TargetName::MAIN]).unwrap(); + sync.need(snapshot, &[]).unwrap(); + // Versions the server's edits never reach. + let ours = (1..=u64::from(SAMPLES)) + .map(|version| snapshot_into(&store, Some(1_000 + version))) + .collect(); + Setup { + _dirs: [files, server, client], + _server: runtime, + url, + node, + tip, + snapshot, + theirs, + client: store, + ours, + } +}); + +/// Serves `service` over HTTP on localhost as `demo`, on a runtime of its +/// own. +fn serve(service: OriginService) -> (Runtime, RemoteUrl) { + let runtime = Runtime::new().unwrap(); + let mut repositories = Repositories::default(); + repositories.insert("demo".parse().unwrap(), service); + let base = runtime.block_on(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, Api::router(repositories)).await }); + format!("http://{address}") + }); + let url = RemoteUrl::Http { + base, + repo: "demo".parse().unwrap(), + }; + (runtime, url) +} + +/// Lands a change with `snapshot` on `main`, returning the acceptance. +fn land(service: &OriginService, snapshot: TreeId) -> Base { + let store = service.store(); + let tip = store.head(&TargetName::MAIN).unwrap().unwrap().acceptance; + let (description, revision) = revision(Base::Acceptance(tip), snapshot); + let change = revision.decode_as::().unwrap().change; + let pack = Pack { + objects: vec![description, revision], + }; + service.put_records(&Caller::Owner, pack).unwrap(); + let request = LandRequest { + proposal: Proposal::Change(change), + onto: None, + }; + let landed = service.land(&Caller::Owner, &request).unwrap(); + let [Landing::Landed { acceptance, .. }] = landed.landings[..] else { + panic!("{landed:?}") + }; + Base::Acceptance(acceptance.acceptance) +} + +/// A revision of a new change with `snapshot`, on `base`, and its +/// description. +fn revision(base: Base, snapshot: TreeId) -> (EncodedObject, EncodedObject) { + let description = EncodedObject::blob(b"bench"); + let revision = Revision { + change: ChangeId::random().unwrap(), + predecessors: Vec::new(), + base, + deps: Vec::new(), + snapshot, + description: description.id().into(), + author: "user:bench".parse().unwrap(), + } + .encode(); + (description, revision) +} + +impl Setup { + fn remote(&self) -> Remote { + Remote::open(&self.url, self.node.clone()).unwrap() + } +} + +/// The snapshot a clone fetches: every file, into an empty repository. +#[divan::bench(sample_count = 10)] +fn clone_snapshot(bencher: Bencher<'_, '_>) { + let setup = &*SETUP; + let remote = setup.remote(); + bencher + .with_inputs(|| { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + (dir, store) + }) + .bench_local_refs(|(_, store)| { + let sync = Sync::new(store, &remote, "laptop".parse().unwrap()); + assert!(sync.need(setup.snapshot, &[]).unwrap()); + }); +} + +/// Fetching a snapshot with 100 files changed since the one the client +/// has. +#[divan::bench(sample_count = SAMPLES)] +fn fetch_100_edited(bencher: Bencher<'_, '_>) { + let setup = &*SETUP; + let remote = setup.remote(); + let sync = Sync::new(&setup.client, &remote, "laptop".parse().unwrap()); + let next = AtomicUsize::new(0); + bencher + .with_inputs(|| setup.theirs[next.fetch_add(1, Ordering::Relaxed) % SAMPLES as usize]) + .bench_local_values(|snapshot| { + sync.need(snapshot, &[setup.snapshot]).unwrap(); + }); +} + +/// Pushing a new change with 100 files edited on `main`'s tip: its +/// revision, and the trees and blobs it changed. +#[divan::bench(sample_count = SAMPLES)] +fn push_100_edited(bencher: Bencher<'_, '_>) { + let setup = &*SETUP; + let remote = setup.remote(); + let sync = Sync::new(&setup.client, &remote, "laptop".parse().unwrap()); + let next = AtomicUsize::new(0); + bencher + .with_inputs(|| { + let snapshot = setup.ours[next.fetch_add(1, Ordering::Relaxed) % SAMPLES as usize]; + let (description, revision) = revision(setup.tip, snapshot); + let change = revision.decode_as::().unwrap().change; + setup.client.write_all(&[description, revision]).unwrap(); + change + }) + .bench_local_values(|change| { + let pushed = sync.push(&[change]).unwrap(); + assert_eq!(pushed.revisions, 1); + }); +} + +/// The changes of the long history a fetch that brings nothing runs on. +const HISTORY: u32 = 10_000; + +/// A server whose repository has [`HISTORY`] changes, each with a revision +/// and a statement, and a client that fetched them all, with the remote's +/// event cursor as of then. +struct History { + _dirs: [tempfile::TempDir; 2], + _server: Runtime, + client: Store, + remote: Remote, + cursor: Cursor, +} + +static HISTORY_SETUP: LazyLock = LazyLock::new(|| { + let [server, client] = [(); 2].map(|()| tempfile::tempdir().unwrap()); + let node = Node { + name: "hosted".parse().unwrap(), + key: Arc::new(NodeKey::from_bytes(&[7; 32])), + }; + let policy = Policy::initial(&node.name, &node.key.public()); + let service = OriginService::create(server.path().join("demo"), node.clone(), &policy).unwrap(); + let tip = service.store().head(&TargetName::MAIN).unwrap().unwrap(); + let state = service.store().get(tip.acceptance).unwrap().body().state; + let description = EncodedObject::blob(b"bench"); + let mut objects = vec![description.clone()]; + let mut changes = Vec::new(); + for change in 0..HISTORY { + let mut bytes = [0; 16]; + bytes[..4].copy_from_slice(&change.to_le_bytes()); + let change = ChangeId::from_bytes(bytes); + let revision = Revision { + change, + predecessors: Vec::new(), + base: Base::Acceptance(tip.acceptance), + deps: Vec::new(), + snapshot: state, + description: description.id().into(), + author: "user:bench".parse().unwrap(), + } + .encode(); + let statement = Statement { + subject: RecordId::Revision(revision.id().into()), + kind: Kind::new("ci.test").unwrap(), + verdict: Some(Verdict::Pass), + issuer: "user:bench".parse().unwrap(), + inputs: None, + payload: None, + supersedes: None, + promoted_from: None, + } + .encode(); + objects.extend([revision, statement]); + changes.push(change); + } + service + .put_records(&Caller::Owner, Pack { objects }) + .unwrap(); + let (runtime, url) = serve(service); + + let store = Store::new(client.path().join(".origin"), Native::default()); + store.init().unwrap(); + let remote = Remote::open(&url, node).unwrap(); + let sync = Sync::new(&store, &remote, "laptop".parse().unwrap()); + let fetch = sync.fetch(&[TargetName::MAIN], &changes, None).unwrap(); + assert_eq!(fetch.fetched.revisions, HISTORY as usize); + let cursor = fetch.cursor; + History { + _dirs: [server, client], + _server: runtime, + client: store, + remote, + cursor, + } +}); + +/// A fetch that brings nothing, from a remote with a long history, with +/// the remote's event cursor as of the last fetch: it costs the same +/// however long the history. +#[divan::bench(sample_count = 20)] +fn fetch_unchanged(bencher: Bencher<'_, '_>) { + let history = &*HISTORY_SETUP; + let sync = Sync::new(&history.client, &history.remote, "laptop".parse().unwrap()); + bencher.bench_local(|| { + let fetch = sync.fetch(&[], &[], Some(&history.cursor)).unwrap(); + assert_eq!(fetch.fetched, Fetched::default()); + }); +} + +/// The same fetch without a cursor, as when it expired, and as every +/// fetch was before cursors: it compares every change. +#[divan::bench(sample_count = 10)] +fn fetch_unchanged_without_cursor(bencher: Bencher<'_, '_>) { + let history = &*HISTORY_SETUP; + let sync = Sync::new(&history.client, &history.remote, "laptop".parse().unwrap()); + bencher.bench_local(|| { + let fetch = sync.fetch(&[], &[], None).unwrap(); + assert_eq!(fetch.fetched, Fetched::default()); + }); +} diff --git a/apps/origin-src/src/cli.rs b/apps/origin-src/src/cli.rs index 87d4a5a5..1499ea1e 100644 --- a/apps/origin-src/src/cli.rs +++ b/apps/origin-src/src/cli.rs @@ -10,7 +10,7 @@ use anyhow::{Context, Result}; use clap::error::ErrorKind; use clap::{Args, CommandFactory, Parser, Subcommand}; use origin_shared::policy::KindPattern; -use origin_shared::{Digest, Inputs, Kind, RepoPath, TargetName, Verdict}; +use origin_shared::{ChangeId, Digest, Inputs, Kind, RepoPath, TargetName, Verdict}; use crate::config::UserConfig; use crate::oplog::Actor; @@ -33,6 +33,18 @@ enum Command { #[arg(default_value = ".")] path: PathBuf, }, + /// Copy a remote repository: the chains of `policy` and a target, and + /// the target's tip, on which a new change starts. Other snapshots are + /// fetched when something reads them. + Clone { + /// The repository: `http://host[:port]/`, or a path. + url: String, + /// Where to create it: by default, named after the repository. + path: Option, + /// The target to follow and start on. + #[arg(short, long, default_value = "main")] + target: TargetName, + }, /// Serve repositories over HTTP, as their authority. Serve { /// The server configuration file. @@ -166,9 +178,6 @@ enum WorkspaceCommand { Target(TargetCommand), /// List the targets, with their tips, authorities and requirements. Targets, - /// Move the working copy where its change went from another - /// workspace, or onto its target's new tip, with the edits made since. - Pull, /// Manage workspaces: working copies sharing this repository. #[command(subcommand)] Workspace(WorkspacesCommand), @@ -178,8 +187,34 @@ enum WorkspaceCommand { /// Move this workspace's working copy back to where it was before its /// latest operation. Records and targets stay as they are. Undo, + /// Show the remote, or set it. + Remote { + /// The new remote: `http://host[:port]/`, or a path. + url: Option, + }, + /// Fetch the remote's new acceptances on the targets followed, and the + /// records of the changes this repository knows. Changes no files. + Fetch { + /// Follow this target too. + #[arg(long)] + target: Vec, + /// Fetch this change too, by its full id. + #[arg(long)] + change: Vec, + }, + /// Fetch, if there's a remote, then move the working copy where its + /// change went, or onto its target's new tip, with the edits made + /// since. + Pull, + /// Share changes with the remote, with the changes below them: their + /// revisions, files and statements. Lands nothing. + Push { + /// The changes: the working copy's by default. + changes: Vec, + }, /// Print events as they happen, one a line, starting with the cursor to - /// resume from. Runs until interrupted. + /// resume from: the remote's, if the repository has one. Runs until + /// interrupted. Watch { /// Only events of this name, such as `check.requested`, or about /// statements of this kind, such as `core.conflict` or `ci.*`. @@ -192,6 +227,9 @@ enum WorkspaceCommand { /// Start after this cursor instead of now: `0` for every event kept. #[arg(long)] since: Option, + /// This node's events, rather than the remote's. + #[arg(long)] + local: bool, }, } @@ -449,7 +487,7 @@ impl Cli { Command::Repo(RepoCommand::Create { path, config }) => { return ServerConfig::load(&config)?.create_repo(&path); } - Command::Init { .. } | Command::Workspace(_) => {} + Command::Init { .. } | Command::Clone { .. } | Command::Workspace(_) => {} } let config = UserConfig::load()?; // The op log records the command line, without the program's name, @@ -464,6 +502,9 @@ impl Cli { Command::Init { path } => { Workspace::init(&path, config, &actor)?; } + Command::Clone { url, path, target } => { + Workspace::clone(&url, path.as_deref(), &target, config, &actor)?; + } Command::Workspace(command) => command.run(config, &actor)?, Command::Serve { .. } | Command::Repo(_) => unreachable!("run above"), } @@ -513,7 +554,6 @@ impl WorkspaceCommand { workspace.target_new(&name, &from)?; } Self::Targets => workspace.targets()?, - Self::Pull => workspace.pull()?, Self::Workspace(WorkspacesCommand::New { change, path }) => { workspace.add_workspace(change.as_deref(), path.as_deref())?; } @@ -521,11 +561,16 @@ impl WorkspaceCommand { Self::Workspace(WorkspacesCommand::Rm { path }) => workspace.remove_workspace(&path)?, Self::Op(OpCommand::Log { limit }) => workspace.op_log(limit)?, Self::Undo => workspace.undo()?, + Self::Remote { url } => workspace.set_remote(url.as_deref())?, + Self::Fetch { target, change } => workspace.fetch(&target, &change)?, + Self::Pull => workspace.pull()?, + Self::Push { changes } => workspace.push(&changes)?, Self::Watch { kind, subject, since, - } => workspace.watch(kind, subject, since)?, + local, + } => workspace.watch(kind, subject, since, local)?, } Ok(()) } diff --git a/apps/origin-src/src/events.rs b/apps/origin-src/src/events.rs index 49fab4a3..f5d51c9f 100644 --- a/apps/origin-src/src/events.rs +++ b/apps/origin-src/src/events.rs @@ -121,6 +121,7 @@ impl<'a, P: Platform> Events<'a, P> { let mut page = Page { events: Vec::new(), next: cursor, + ..Page::default() }; let oplog = self.store.oplog(); let read = oplog.after(cursor)?; @@ -457,7 +458,8 @@ mod tests { held, Page { events: Vec::new(), - next: crashed - 1 + next: crashed - 1, + ..Page::default() }, "it may be in progress" ); @@ -520,7 +522,8 @@ mod tests { empty, Page { events: Vec::new(), - next: end + next: end, + ..Page::default() } ); thread::scope(|scope| { diff --git a/apps/origin-src/src/http.rs b/apps/origin-src/src/http.rs index f028238a..01579d67 100644 --- a/apps/origin-src/src/http.rs +++ b/apps/origin-src/src/http.rs @@ -15,12 +15,13 @@ use axum::response::{IntoResponse, Response}; use origin_shared::ChangeId; use origin_shared::Head; use origin_shared::api::{ - Annotation, ApiError, ErrorCode, EventsQuery, FetchRecords, LandRequest, Landings, LogEntry, - LogQuery, NewTarget, RepoName, RepoSummary, Revisions, StatementsQuery, Stored, TargetInfo, + Annotation, ApiError, ErrorCode, EventsQuery, FetchRecords, FetchSnapshot, Inventory, + InventoryRequest, LandRequest, Landings, LogEntry, LogQuery, MORE, NewTarget, PutQuery, + RepoName, RepoSummary, Revisions, StatementsQuery, Stored, TargetInfo, }; use origin_shared::event::Page; use origin_shared::landing::Pending; -use origin_shared::pack::{Pack, PackError}; +use origin_shared::pack::{Pack, PackError, Part}; use utoipa::OpenApi; use utoipa_axum::router::OpenApiRouter; use utoipa_axum::routes; @@ -44,14 +45,19 @@ type Repos = State>; pub struct Api; impl Api { - /// The largest request accepted, which bounds a batch of records put. + /// The largest request accepted, which bounds an object put: clients + /// split larger pushes into requests of at most + /// [`Pack::LIMIT`](origin_shared::pack::Pack::LIMIT), so only a single + /// object larger than that takes a request of its own this large. const MAX_REQUEST: usize = 1 << 30; fn documented() -> OpenApiRouter> { OpenApiRouter::with_openapi(ApiDoc::openapi()) .routes(routes!(list_repos)) .routes(routes!(fetch_records)) + .routes(routes!(fetch_snapshot)) .routes(routes!(put_records)) + .routes(routes!(inventory)) .routes(routes!(statements)) .routes(routes!(revisions)) .routes(routes!(targets, create_target)) @@ -160,7 +166,28 @@ async fn list_repos(State(repositories): Repos, caller: Caller) -> Json Response { + let Part { pack, more } = self.0; + let mut response = ( + [(header::CONTENT_TYPE, "application/octet-stream")], + pack.encode(), + ) + .into_response(); + if more { + response + .headers_mut() + .insert(MORE, header::HeaderValue::from_static("true")); + } + response + } +} + +/// Fetches records by id, each with the blob it carries, as a pack. #[utoipa::path( post, path = "/api/v1/repos/{repo}/records/fetch", @@ -168,8 +195,9 @@ async fn list_repos(State(repositories): Repos, caller: Caller) -> Json, caller: Caller, Json(request): Json, -) -> Result { - let pack = call(repos, repo, caller, move |service, caller| { +) -> Result { + call(repos, repo, caller, move |service, caller| { service.records(caller, &request.ids) }) - .await?; - Ok(( - [(header::CONTENT_TYPE, "application/octet-stream")], - pack.encode(), + .await + .map(PackResponse) +} + +/// Fetches the objects of a snapshot the caller lacks, as a pack: the only +/// way trees and blobs leave a repository. +#[utoipa::path( + post, + path = "/api/v1/repos/{repo}/snapshots/fetch", + operation_id = "fetch_snapshot", + params(("repo" = String, Path, description = "The repository")), + request_body = FetchSnapshot, + responses( + (status = 200, description = "The trees and blobs, parents before children, as a pack, of at most 64 MiB, but for an object alone", content_type = "application/octet-stream", body = Binary, headers(("origin-more" = bool, description = "Present if the pack stopped short: ask again, skipping the objects it holds"))), + (status = 404, description = "The repository or the snapshot does not exist", body = ApiError) ) - .into_response()) +)] +async fn fetch_snapshot( + repos: Repos, + Path(repo): Path, + caller: Caller, + Json(request): Json, +) -> Result { + call(repos, repo, caller, move |service, caller| { + service.snapshot(caller, &request) + }) + .await + .map(PackResponse) } /// Puts a batch of objects, as a pack: snapshots, revisions and /// statements, each needing only objects the batch or the repository has, -/// of the kinds it names. +/// of the kinds it names. With `expect`, only if the changes it names have +/// the heads it says when the records are published. #[utoipa::path( put, path = "/api/v1/repos/{repo}/records", operation_id = "put_records", - params(("repo" = String, Path, description = "The repository")), + params(("repo" = String, Path, description = "The repository"), PutQuery), request_body(content = Binary, content_type = "application/octet-stream", description = "The objects, as a pack"), responses( (status = 200, description = "The objects are durable", body = Stored), (status = 400, description = "The pack is invalid, holds an acceptance, or needs objects the repository doesn't have, or has as another kind", body = ApiError), - (status = 404, description = "The repository does not exist", body = ApiError) + (status = 404, description = "The repository does not exist", body = ApiError), + (status = 409, description = "A change of `expect` has other heads now", body = ApiError) ) )] async fn put_records( repos: Repos, Path(repo): Path, + Query(query): Query, caller: Caller, body: Bytes, ) -> Result, HttpError> { let pack = Pack::decode(&body).map_err(HttpError::Pack)?; call(repos, repo, caller, move |service, caller| { - service.put_records(caller, pack) + service.push(caller, pack, &query.expect.0) + }) + .await + .map(Json) +} + +/// Lists what the repository holds of some changes: their revisions, +/// heads and statements, by id. +#[utoipa::path( + post, + path = "/api/v1/repos/{repo}/changes/inventory", + operation_id = "inventory", + params(("repo" = String, Path, description = "The repository")), + request_body = InventoryRequest, + responses( + (status = 200, description = "An inventory for each change, in the order asked: empty for a change the repository never heard of", body = Vec), + (status = 404, description = "The repository does not exist", body = ApiError) + ) +)] +async fn inventory( + repos: Repos, + Path(repo): Path, + caller: Caller, + Json(request): Json, +) -> Result>, HttpError> { + call(repos, repo, caller, move |service, caller| { + service.inventory(caller, &request) }) .await .map(Json) @@ -334,7 +413,7 @@ async fn log( Query(query): Query, ) -> Result>, HttpError> { call(repos, repo, caller, move |service, caller| { - service.log(caller, &query.target, query.since) + service.log(caller, &query) }) .await .map(Json) diff --git a/apps/origin-src/src/index.rs b/apps/origin-src/src/index.rs index 9ed04cda..3845606c 100644 --- a/apps/origin-src/src/index.rs +++ b/apps/origin-src/src/index.rs @@ -382,6 +382,11 @@ impl Index { &self.arrivals } + /// Every change with a revision, in no particular order. + pub fn changes(&self) -> impl Iterator { + self.changes.keys().copied() + } + /// The revisions of `change`, in arrival order. #[must_use] pub fn revisions(&self, change: ChangeId) -> &[RevisionId] { diff --git a/apps/origin-src/src/lib.rs b/apps/origin-src/src/lib.rs index 374172ed..092361e6 100644 --- a/apps/origin-src/src/lib.rs +++ b/apps/origin-src/src/lib.rs @@ -16,9 +16,11 @@ pub mod oplog; pub mod output; pub mod policy; pub mod queue; +pub mod remote; pub mod server; pub mod service; pub mod store; +pub mod sync; pub mod targets; pub mod tree; pub mod workspace; diff --git a/apps/origin-src/src/oplog.rs b/apps/origin-src/src/oplog.rs index 1e75f90c..4aea9a2f 100644 --- a/apps/origin-src/src/oplog.rs +++ b/apps/origin-src/src/oplog.rs @@ -17,6 +17,7 @@ //! //! ```text //! .origin/oplog/ +//! generation a random id, made with the log //! lock held exclusively while appending //! writers/ held by each writer while it writes //! segments of entries, 16 hex digits each @@ -25,7 +26,9 @@ //! A segment is a file of [frames](crate::frames), an entry each. Entries are //! appended to the last segment until it's full, then to a new one, and the //! oldest segments are deleted beyond a cap: the log keeps the latest 16 MiB -//! of entries, and a cursor older than that has expired. +//! of entries, and a cursor older than that has expired. So has a cursor +//! of another log, such as the one a restored repository had: the log's +//! generation tells them apart ([`OpLog::generation`]). //! //! **Writing.** Events and the index come from the entries, so every entry //! must be durable before a reader moves past it: a power loss could drop @@ -455,17 +458,56 @@ impl<'a, P: Platform> OpLog<'a, P> { let writers = self.dir().join(Self::WRITERS); fs::create_dir_all(&writers).map_err(|source| StoreError::io(&writers, source))?; self.start_segment(1, &[])?; + self.generation()?; Ok(()) } - /// Opens and locks one of the log's lock files, creating the log's - /// directory, durably, if it's missing. - fn lock(&self, name: &str, mode: LockMode) -> Result { + /// The file holding the log's generation. + const GENERATION: &str = "generation"; + + /// The log's generation: a random id made with it, which a log made + /// later, as when a repository is restored from a copy of its records, + /// doesn't share. The cursors of one log mean nothing in another. + /// + /// # Errors + /// + /// Fails if the generation cannot be read, or made and written. + pub fn generation(&self) -> Result { + let path = self.dir().join(Self::GENERATION); + let read = |path: &Path| match fs::read_to_string(path) { + Ok(generation) => Ok(Some(generation.trim_end().to_owned())), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(StoreError::io(path, source)), + }; + if let Some(generation) = read(&path)? { + return Ok(generation); + } + // A log made without one. Writers racing to make it each write + // theirs, and the last stays: a cursor from another only costs a + // full read. + self.create_dir()?; + let generation = getrandom::u64().map_err(StoreError::Random)?; + let write = self.store.durable_write(); + write.stage(&path, format!("{generation:016x}\n").as_bytes())?; + write.commit()?; + Ok(read(&path)?.unwrap_or_default()) + } + + /// Creates the log's directory, durably, if it's missing. + fn create_dir(&self) -> Result<(), StoreError> { let dir = self.dir(); if !dir.is_dir() { fs::create_dir_all(&dir).map_err(|source| StoreError::io(&dir, source))?; self.store.sync_dir(self.store.root())?; } + Ok(()) + } + + /// Opens and locks one of the log's lock files, creating the log's + /// directory, durably, if it's missing. + fn lock(&self, name: &str, mode: LockMode) -> Result { + self.create_dir()?; + let dir = self.dir(); Ok(LockedFile::open( &self.store.platform, &dir.join(name), diff --git a/apps/origin-src/src/queue.rs b/apps/origin-src/src/queue.rs index a8860c35..ec0eea8e 100644 --- a/apps/origin-src/src/queue.rs +++ b/apps/origin-src/src/queue.rs @@ -45,6 +45,7 @@ use std::collections::HashSet; use std::fs; use std::io; use std::path::PathBuf; +use std::slice; use origin_platform::{DurableWrite, LockMode, LockedFile, Native, Platform}; use origin_shared::landing::{Blocked, Landing, Pending, Proposal}; @@ -501,6 +502,9 @@ impl Session<'_, '_, P> { revision: None, }); } + // Held until the rebased revision is published, so no push makes + // the change diverge meanwhile. + let _publishing = self.publishing(revision.change, id)?; let rebased = Revision { predecessors: vec![id], base: tip, @@ -521,6 +525,18 @@ impl Session<'_, '_, P> { }) } + /// Takes the lock every publication of revisions takes (see + /// [`Store::lock_pushes`]), once `head` is still the only head of + /// `change`. + fn publishing(&mut self, change: ChangeId, head: RevisionId) -> Result { + let lock = self.store().lock_pushes()?; + self.annotations.refresh()?; + if self.annotations.index().heads(change) != [head] { + return Err(LandError::Moved(change)); + } + Ok(lock) + } + /// Checks the policy for `candidate`, and lands it if its evidence is /// there, or keeps it waiting. Writes the records in layers, each /// durable before the next refers to it: the trees, then the rebased @@ -599,9 +615,8 @@ impl Session<'_, '_, P> { if Policy::governing(encoded.id().into(), acceptance.body())? != self.policy_head { return Err(PolicyError::NotPredecessor(acceptance.body().seq).into()); } - records.push(encoded); - store.write_all(&records)?; - let id = store.append(&self.onto, &acceptance)?; + let id = encoded.id().into(); + store.append_all(&self.onto, slice::from_ref(&acceptance), &records)?; self.tip = Head { seq: acceptance.body().seq, acceptance: id, @@ -864,6 +879,9 @@ pub enum LandError { /// Its heads. heads: Vec, }, + /// A revision of the change was published while it was rebased. + #[error("change {} moved while it was landing: land it again", .0.short())] + Moved(ChangeId), /// The change was given up. #[error("change {} was abandoned, so it cannot land", .0.short())] Abandoned(ChangeId), diff --git a/apps/origin-src/src/remote.rs b/apps/origin-src/src/remote.rs new file mode 100644 index 00000000..e9abffb9 --- /dev/null +++ b/apps/origin-src/src/remote.rs @@ -0,0 +1,245 @@ +//! Remotes: the repositories a node shares its work with. +//! +//! A remote is a server over HTTP, `http://host[:port]/`, or a +//! repository on a local path: a server's repository, or a working copy's. +//! A local one is served by an in-process [`OriginService`] run by this +//! node, so both answer the same calls, and [`sync`](crate::sync) never +//! tells them apart. Calls block: an HTTP remote runs the SDK on a runtime +//! of its own. + +use std::fmt; +use std::path::{self, Path, PathBuf}; + +use origin_client::Client; +use origin_shared::Head; +use origin_shared::api::{ + ErrorCode, EventsQuery, FetchRecords, FetchSnapshot, Inventory, InventoryRequest, LandRequest, + Landings, LogEntry, LogQuery, NewTarget, PutQuery, RepoName, Stored, TargetInfo, +}; +use origin_shared::event::Page; +use origin_shared::landing::Pending; +use origin_shared::pack::{Pack, Part}; + +use crate::service::{Caller, Node, OriginService, ServiceError}; +use crate::workspace::Workspace; + +/// A repository to share work with. +#[derive(Debug)] +pub enum Remote { + /// A repository on a server. + Http { + /// The runtime the SDK's requests run on. + runtime: tokio::runtime::Runtime, + /// The repository. + repo: origin_client::Repo, + }, + /// A repository on a local path, served in-process. + Local(Box), +} + +/// Defines the remote's calls, each answered by an SDK method over HTTP, or +/// by a service method in-process, called as the node's owner. +macro_rules! calls { + ($( + $(#[doc = $doc:literal])* + fn $name:ident($($arg:ident: $type:ty),*) -> $output:ty = + $http:ident($($http_arg:expr),*), $local:ident($($local_arg:expr),*); + )*) => { + impl Remote {$( + $(#[doc = $doc])* + /// + /// # Errors + /// + /// Fails if the remote cannot be reached or refuses. + pub fn $name(&self, $($arg: $type),*) -> Result<$output, RemoteError> { + match self { + Self::Http { runtime, repo } => { + Ok(runtime.block_on(repo.$http($($http_arg),*))?) + } + Self::Local(service) => Ok(service.$local(&Caller::Owner, $($local_arg),*)?), + } + } + )*} + }; +} + +calls! { + /// Every target, with its tip, authority and requirements. + fn targets() -> Vec = targets(), targets(); + /// The acceptances of a target, oldest first. + fn log(query: &LogQuery) -> Vec = log(query), log(query); + /// Records by id, each followed by the blob it carries: those up to the + /// last record of the part, if it stops short. + fn records(request: &FetchRecords) -> Part = fetch_records(request), records(&request.ids); + /// The trees and blobs of a snapshot that the request's snapshots lack, + /// in parts. + fn snapshot(request: &FetchSnapshot) -> Part = fetch_snapshot(request), snapshot(request); + /// What the remote holds of some changes. + fn inventory(request: &InventoryRequest) -> Vec = + inventory(request), inventory(request); + /// Puts a batch of objects. + fn put_records(pack: Pack) -> Stored = put_records(&pack), put_records(pack); + /// Puts a batch of objects, if the changes the query names have the + /// heads it says. + fn push(pack: Pack, query: &PutQuery) -> Stored = + push_records(&pack, query), push(pack, &query.expect.0); + /// Submits a landing to the remote's queue. + fn land(request: &LandRequest) -> Landings = land(request), land(request); + /// The landings that wait on the remote. + fn landings() -> Vec = landings(), landings(); + /// Creates a target the remote orders. + fn create_target(request: &NewTarget) -> Head = create_target(request), create_target(request); + /// The events after a cursor, waiting for some. + fn events(query: &EventsQuery) -> Page = events(query), events(query); +} + +impl Remote { + /// Opens the remote at `url`: an `http://` URL whose last path segment + /// names the repository, or the path of a repository, which `node` + /// serves in-process. + /// + /// # Errors + /// + /// Fails if the URL names no repository, or the path holds none. + pub fn open(url: &RemoteUrl, node: Node) -> Result { + match url { + RemoteUrl::Http { base, repo } => Ok(Self::Http { + runtime: tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(RemoteError::Runtime)?, + repo: Client::new(base).repo(repo.clone()), + }), + RemoteUrl::Path(path) => { + // A working copy's repository is in its `.origin`. + let meta = path.join(Workspace::META_DIR); + let root = if meta.is_dir() { meta } else { path.clone() }; + let service = OriginService::open(root, node) + .map_err(|source| RemoteError::Open(Box::new(source)))?; + Ok(Self::Local(Box::new(service))) + } + } + } +} + +/// Where a remote is. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RemoteUrl { + /// A repository on a server. + Http { + /// The server, such as `http://127.0.0.1:7878`. + base: String, + /// The repository. + repo: RepoName, + }, + /// A repository on a local path, absolute. + Path(PathBuf), +} + +impl RemoteUrl { + /// Parses a remote URL: `http://host[:port]/`, or a path, + /// made absolute, so the URL works from anywhere. + /// + /// # Errors + /// + /// Fails if an HTTP URL names no repository, or is HTTPS. + pub fn parse(url: &str) -> Result { + if url.starts_with("https://") { + return Err(RemoteError::HttpsUnsupported); + } + let Some(rest) = url.strip_prefix("http://") else { + let path = path::absolute(Path::new(url)) + .map_err(|_| RemoteError::InvalidUrl(url.to_owned()))?; + return Ok(Self::Path(path)); + }; + let invalid = || RemoteError::InvalidUrl(url.to_owned()); + let (host, name) = rest + .trim_end_matches('/') + .rsplit_once('/') + .ok_or_else(invalid)?; + Ok(Self::Http { + base: format!("http://{host}"), + repo: RepoName::new(name).map_err(|_| invalid())?, + }) + } + + /// The name a clone's directory takes by default: the repository's, or + /// the path's last component. + #[must_use] + pub fn name(&self) -> Option { + match self { + Self::Http { repo, .. } => Some(repo.to_string()), + Self::Path(path) => Some(path.file_name()?.to_string_lossy().into_owned()), + } + } +} + +impl fmt::Display for RemoteUrl { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Http { base, repo } => write!(f, "{base}/{repo}"), + Self::Path(path) => write!(f, "{}", path.display()), + } + } +} + +/// A failure to talk to a remote. +#[derive(Debug, thiserror::Error)] +pub enum RemoteError { + /// The remote URL is not usable. + #[error("invalid remote `{0}`: expected http://host[:port]/ or a path")] + InvalidUrl(String), + /// The client has no TLS support yet. + #[error("HTTPS remotes are not supported yet")] + HttpsUnsupported, + /// The runtime for HTTP requests cannot start. + #[error("cannot start the runtime for HTTP requests")] + Runtime(#[source] std::io::Error), + /// A local path holds no repository to serve. + #[error("cannot open the remote repository")] + Open(#[source] Box), + /// The server cannot be reached or refused. + #[error(transparent)] + Http(#[from] origin_client::Error), + /// The repository on a local path refused. + #[error(transparent)] + Local(#[from] ServiceError), +} + +impl RemoteError { + /// What kind of refusal this is, if the remote refused. + #[must_use] + pub fn code(&self) -> Option { + match self { + Self::Http(origin_client::Error::Api(error)) => Some(error.code), + Self::Local(error) => Some(error.code()), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_urls() { + let url = RemoteUrl::parse("http://localhost:7878/demo/").unwrap(); + assert_eq!( + url, + RemoteUrl::Http { + base: "http://localhost:7878".to_owned(), + repo: RepoName::new("demo").unwrap(), + } + ); + assert_eq!(url.to_string(), "http://localhost:7878/demo"); + assert_eq!(url.name().as_deref(), Some("demo")); + assert!(RemoteUrl::parse("http://localhost").is_err()); + assert!(RemoteUrl::parse("http://localhost/.hidden").is_err()); + assert!(RemoteUrl::parse("https://localhost/demo").is_err()); + let RemoteUrl::Path(path) = RemoteUrl::parse("server/demo").unwrap() else { + panic!("expected a path"); + }; + assert!(path.is_absolute() && path.ends_with("server/demo")); + } +} diff --git a/apps/origin-src/src/server.rs b/apps/origin-src/src/server.rs index f6b131ce..acb3e1df 100644 --- a/apps/origin-src/src/server.rs +++ b/apps/origin-src/src/server.rs @@ -66,7 +66,11 @@ impl ServerConfig { fs::read_to_string(path).with_context(|| format!("cannot read {}", path.display()))?; let mut config: Self = toml::from_str(&content) .with_context(|| format!("invalid configuration in {}", path.display()))?; - let base = path.parent().unwrap_or(Path::new(".")); + // A file in the current directory has an empty parent. + let base = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); config.key = base.join(&config.key); for repo in config.repos.values_mut() { *repo = base.join(&*repo); diff --git a/apps/origin-src/src/service.rs b/apps/origin-src/src/service.rs index 096ebd4b..a8801d9a 100644 --- a/apps/origin-src/src/service.rs +++ b/apps/origin-src/src/service.rs @@ -19,22 +19,24 @@ use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; +use std::iter; use std::path::PathBuf; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Duration; use origin_platform::Native; use origin_shared::api::{ - Annotation, ErrorCode, EventsQuery, LandRequest, Landings, LogEntry, NewTarget, RevisionEntry, - Revisions, Stored, TargetInfo, + Annotation, ErrorCode, EventsQuery, Expected, FetchSnapshot, Inventory, InventoryRequest, + LandRequest, Landings, LogEntry, LogQuery, NewTarget, RevisionEntry, Revisions, Stored, + TargetInfo, }; use origin_shared::event::{Filter, Page}; use origin_shared::landing::{Pending, Proposal}; use origin_shared::object::DecodeError; -use origin_shared::pack::Pack; +use origin_shared::pack::{Pack, Packer, Part}; use origin_shared::{ Base, ChangeId, Head, NodeKey, Object, ObjectId, ObjectKind, Principal, PrincipalName, - RecordId, Statement, StatementId, TargetName, + RecordId, Statement, StatementId, TargetName, TreeId, }; use crate::annotations::{AnnotateError, Annotations}; @@ -79,6 +81,8 @@ pub struct OriginService { /// The record indexes kept open, but those in use: see the module. indexes: Mutex>, wakeup: Wakeup, + /// The most bytes a fetch's answer holds: [`Pack::LIMIT`], but in tests. + pack_limit: usize, } impl OriginService { @@ -99,6 +103,16 @@ impl OriginService { node, indexes: Mutex::default(), wakeup: Wakeup::default(), + pack_limit: Pack::LIMIT, + } + } + + /// The service, answering fetches in parts of at most `limit` bytes. + #[cfg(test)] + pub(crate) fn with_pack_limit(self, limit: usize) -> Self { + Self { + pack_limit: limit, + ..self } } @@ -179,17 +193,138 @@ impl OriginService { self.indexes.lock().unwrap_or_else(PoisonError::into_inner) } - /// The objects `ids` name, in their order, as a pack. + /// The records `ids` name, in their order, as a pack: revisions, + /// statements and acceptances, each followed by the blob it carries, if + /// the repository has it: a revision's description, a statement's + /// payload. Snapshots only leave through [`OriginService::snapshot`]. + /// The pack stops short at [`Pack::LIMIT`], before a record that + /// doesn't fit with its blob: the rest comes from asking for the ids + /// after the last record it has. + /// + /// # Errors + /// + /// Fails if the repository doesn't have one, one is a blob or a tree, or + /// it cannot read one. + pub fn records(&self, _caller: &Caller, ids: &[ObjectId]) -> Result { + let mut packer = Packer::new(self.pack_limit); + for &id in ids { + let object = self.store.read(id)?; + let carried = match object + .decode() + .map_err(|source| StoreError::Decode { id, source })? + { + Object::Blob(_) | Object::Tree(_) => return Err(ServiceError::NotARecord(id)), + Object::Revision(revision) => Some(revision.description.into()), + Object::Statement(statement) => statement.payload.map(Into::into), + Object::Acceptance(_) => None, + }; + let carried = carried + .map(|id| self.store.try_read(id)) + .transpose()? + .flatten(); + let group: Vec<_> = iter::once(object).chain(carried).collect(); + if !packer.fits(&group) { + return Ok(Part { + pack: packer.finish(), + more: true, + }); + } + packer.extend(group); + } + Ok(Part { + pack: packer.finish(), + more: false, + }) + } + + /// The objects of the snapshot the request names that the caller lacks, + /// as a pack, parents before children: those none of the snapshots it + /// has holds at the same path, under the paths it asks for, but the + /// first `skip`. See [`Store::added`]. The pack stops short at + /// [`Pack::LIMIT`]: the rest comes from asking again, skipping what + /// came. /// /// # Errors /// - /// Fails if the repository doesn't have one, or cannot read it. - pub fn records(&self, _caller: &Caller, ids: &[ObjectId]) -> Result { - let objects = ids + /// Fails if the repository doesn't have the snapshot, or cannot read + /// it. + pub fn snapshot( + &self, + _caller: &Caller, + request: &FetchSnapshot, + ) -> Result { + let is_tree = |id: TreeId| -> Result { + let object = self.store.try_read(id.into())?; + Ok(object.is_some_and(|object| object.kind().ok() == Some(ObjectKind::Tree))) + }; + if !is_tree(request.snapshot)? { + return Err(ServiceError::UnknownSnapshot(request.snapshot)); + } + let mut have = Vec::new(); + for &id in &request.have { + if is_tree(id)? { + have.push(id); + } + } + let added = self.store.added(request.snapshot, &have, &request.paths)?; + let skip = usize::try_from(request.skip).unwrap_or(usize::MAX); + let ids: Vec = added .iter() - .map(|&id| self.store.read(id)) - .collect::>()?; - Ok(Pack { objects }) + .skip(skip) + .map(|placed| placed.entry.id) + .collect(); + let mut packer = Packer::new(self.pack_limit); + let mut more = false; + self.store.read_each(&ids, |object| { + more = !packer.fits([&object]); + if !more { + packer.extend([object]); + } + Ok::<_, StoreError>(!more) + })?; + Ok(Part { + pack: packer.finish(), + more, + }) + } + + /// What the repository holds of each change: its revisions and heads, + /// and the statements about them, by id. A change it never heard of + /// has none. + /// + /// # Errors + /// + /// Fails if the index cannot be read. + pub fn inventory( + &self, + _caller: &Caller, + request: &InventoryRequest, + ) -> Result, ServiceError> { + self.annotations(|annotations| { + let index = annotations.index(); + Ok(request + .changes + .iter() + .map(|&change| { + let revisions = index.revisions(change).to_vec(); + let mut subjects: Vec<_> = + revisions.iter().copied().map(RecordId::Revision).collect(); + let mut statements = Vec::new(); + while let Some(subject) = subjects.pop() { + for &statement in index.statements(subject) { + statements.push(statement); + subjects.push(RecordId::Statement(statement)); + } + } + Inventory { + change, + heads: index.heads(change), + revisions, + statements, + } + }) + .collect()) + }) } /// Writes the objects of `pack`: snapshots, revisions and statements. @@ -210,10 +345,29 @@ impl OriginService { /// /// Fails if an object is malformed, an acceptance, or needs an object /// neither the pack nor the repository has, or one of another kind, if - /// a promotion doesn't - /// match its original, which leaves the other records stored, or if - /// the objects cannot be written. - pub fn put_records(&self, _caller: &Caller, pack: Pack) -> Result { + /// a promotion doesn't match its original, which leaves the other + /// records stored, or if the objects cannot be written. + pub fn put_records(&self, caller: &Caller, pack: Pack) -> Result { + self.push(caller, pack, &[]) + } + + /// Puts `pack` like [`OriginService::put_records`], if each change of + /// `expect` has the heads it names when its records are published: a + /// compare-and-swap, which every other publication of revisions waits + /// for, puts and the landing queue's rebases alike. So of two pushes + /// made from the same view of a change, or a push and a rebase, the + /// second is refused instead of making it diverge. + /// + /// # Errors + /// + /// Fails like [`OriginService::put_records`], or if a change of + /// `expect` moved, which publishes no record, only snapshots. + pub fn push( + &self, + _caller: &Caller, + pack: Pack, + expect: &[Expected], + ) -> Result { // Objects this finds stored stay until what refers to them is. let _held = self.store.hold_objects()?; let held: HashMap> = pack @@ -237,7 +391,7 @@ impl OriginService { let decoded = object .decode() .map_err(|source| ServiceError::Malformed { id, source })?; - for (needed, kind) in Dependencies::of(&decoded) { + for (needed, kind) in Dependencies::of(&decoded).all() { match held.get(&needed) { Some(&held) if held != Some(kind) => { return Err(ServiceError::WrongKind { id, needed, kind }); @@ -275,10 +429,22 @@ impl OriginService { self.check_kinds(&stored)?; dependencies.commit()?; let mut new = 0; + if !snapshots.is_empty() { + new += self.store.write_all(&snapshots)?; + } + // Held until the records are published, so nothing else moves the + // changes in between: see `Store::lock_pushes`. + let _pushing = if !revisions.is_empty() || !expect.is_empty() { + let lock = self.store.lock_pushes()?; + self.check_heads(expect)?; + Some(lock) + } else { + None + }; // Each layer durable before the next, as a workspace writes them: // statements before the revisions they're about, so a crash never // publishes a revision without its `core.conflict`. - for layer in [snapshots, statements, revisions] { + for layer in [statements, revisions] { if !layer.is_empty() { new += self.store.write_all(&layer)?; } @@ -310,6 +476,23 @@ impl OriginService { Ok(Stored { new: new as u64 }) } + /// Checks that each change of `expect` has the heads it names. + fn check_heads(&self, expect: &[Expected]) -> Result<(), ServiceError> { + self.annotations(|annotations| { + for expected in expect { + let heads = annotations.index().heads(expected.change); + let now: HashSet<_> = heads.iter().collect(); + if now != expected.heads.iter().collect() { + return Err(ServiceError::Moved { + change: expected.change, + heads, + }); + } + } + Ok(()) + }) + } + /// Checks that each object of `stored`, which the repository has, is of /// the kind the object of the pack needing it names, reading them from /// every core. @@ -395,24 +578,23 @@ impl OriginService { Ok(targets) } - /// The acceptances of `target` from position `since` on, oldest first. + /// The acceptances of the query's target from position `since` on, + /// oldest first, up to its `limit`. /// /// # Errors /// /// Fails if the target doesn't exist, or its log or an acceptance /// cannot be read. - pub fn log( - &self, - _caller: &Caller, - target: &TargetName, - since: u64, - ) -> Result, ServiceError> { + pub fn log(&self, _caller: &Caller, query: &LogQuery) -> Result, ServiceError> { + let target = &query.target; if self.store.head(target)?.is_none() { return Err(ServiceError::UnknownTarget(target.clone())); } - self.store - .log_since(target, since)? - .into_iter() + let mut ids = self.store.log_since(target, query.since)?; + if let Some(limit) = query.limit { + ids.truncate(usize::try_from(limit).unwrap_or(usize::MAX)); + } + ids.into_iter() .map(|id| { Ok(LogEntry { id, @@ -462,8 +644,7 @@ impl OriginService { landed } - /// The target the stack of `change` is on: that of the acceptance at - /// the bottom of its head's stack. + /// The target the stack of `change` is on: that of its head's. fn target_of(&self, index: &Index, change: ChangeId) -> Result { let head = match index.heads(change)[..] { [head] => head, @@ -476,18 +657,7 @@ impl OriginService { .into()); } }; - let mut base = self.store.get(head)?.base; - let acceptance = loop { - match base { - Base::Revision(id) => base = self.store.get(id)?.base, - Base::Acceptance(id) => break id, - } - }; - let target = self.store.get(acceptance)?.body().target; - Ok(self - .store - .target_name(target)? - .ok_or(StoreError::NotFound(acceptance.into()))?) + Ok(self.store.stack_target(head)?) } /// Every landing that waits, on every target, and why. @@ -525,8 +695,15 @@ impl OriginService { /// # Errors /// /// Fails if the op log no longer keeps the events after the cursor, or - /// it or a record cannot be read. + /// isn't of the generation the query names, or it or a record cannot + /// be read. pub fn events(&self, _caller: &Caller, query: &EventsQuery) -> Result { + let generation = self.store.oplog().generation()?; + if let Some(theirs) = &query.generation + && *theirs != generation + { + return Err(ServiceError::OtherLog(theirs.clone())); + } let events = Events::new(&self.store).waking_on(&self.wakeup); let cursor = match query.cursor { Some(cursor) => cursor, @@ -541,42 +718,56 @@ impl OriginService { .unwrap_or(Self::MAX_EVENTS) .min(Self::MAX_EVENTS) as usize; let wait = query.wait.map_or(Duration::ZERO, |wait| wait.0); - Ok(events.wait(cursor, &filter, limit, wait.min(Self::MAX_WAIT))?) + let page = events.wait(cursor, &filter, limit, wait.min(Self::MAX_WAIT))?; + Ok(Page { generation, ..page }) } } /// What an object needs the repository to have for it to be usable, as -/// [`OriginService::put_records`] lists, and the kind each must be. A +/// [`OriginService::put_records`] lists, and the kind each must be: the +/// records it refers to, and the content, snapshots and blobs. A /// revision's predecessors and a statement's payload are history and /// details: the repository may lack them, and retention may have dropped a -/// payload. -struct Dependencies; +/// payload. An acceptance needs nothing here: its chain brings what it +/// refers to. +#[derive(Debug, Default)] +pub(crate) struct Dependencies { + /// Revisions, statements and acceptances. + pub records: Vec<(ObjectId, ObjectKind)>, + /// Trees and blobs. + pub content: Vec<(ObjectId, ObjectKind)>, +} impl Dependencies { - fn of(object: &Object) -> Vec<(ObjectId, ObjectKind)> { + pub(crate) fn of(object: &Object) -> Self { match object { - Object::Blob(_) | Object::Acceptance(_) => Vec::new(), - Object::Tree(tree) => tree - .iter() - .map(|(_, entry)| (entry.id, entry.kind.object_kind())) - .collect(), + Object::Blob(_) | Object::Acceptance(_) => Self::default(), + Object::Tree(tree) => Self { + records: Vec::new(), + content: tree + .iter() + .map(|(_, entry)| (entry.id, entry.kind.object_kind())) + .collect(), + }, Object::Revision(revision) => { let base = match revision.base { Base::Acceptance(id) => (id.into(), ObjectKind::Acceptance), Base::Revision(id) => (id.into(), ObjectKind::Revision), }; - let mut needed = vec![ - (revision.snapshot.into(), ObjectKind::Tree), - (revision.description.into(), ObjectKind::Blob), - base, - ]; - needed.extend( + let mut records = vec![base]; + records.extend( revision .deps .iter() .map(|&id| (id.into(), ObjectKind::Revision)), ); - needed + Self { + records, + content: vec![ + (revision.snapshot.into(), ObjectKind::Tree), + (revision.description.into(), ObjectKind::Blob), + ], + } } Object::Statement(statement) => Self::of_statement(statement), } @@ -584,16 +775,25 @@ impl Dependencies { /// What a statement needs: its subject, and the statements it /// supersedes or promotes. - fn of_statement(statement: &Statement) -> Vec<(ObjectId, ObjectKind)> { + pub(crate) fn of_statement(statement: &Statement) -> Self { + let mut dependencies = Self::default(); let subject = (statement.subject.into(), statement.subject.kind()); - [Some(subject)] - .into_iter() - .chain( - [statement.supersedes, statement.promoted_from] - .map(|id| id.map(|id| (id.into(), ObjectKind::Statement))), - ) - .flatten() - .collect() + match statement.subject { + RecordId::Snapshot(_) => dependencies.content.push(subject), + _ => dependencies.records.push(subject), + } + dependencies.records.extend( + [statement.supersedes, statement.promoted_from] + .into_iter() + .flatten() + .map(|id| (id.into(), ObjectKind::Statement)), + ); + dependencies + } + + /// Every object needed, and the kind it must be. + pub(crate) fn all(&self) -> impl Iterator { + self.records.iter().chain(&self.content).copied() } } @@ -615,6 +815,12 @@ pub enum ServiceError { /// No target has this name. #[error("no target is named `{0}`")] UnknownTarget(TargetName), + /// The repository has no such snapshot. + #[error("snapshot {0} not found")] + UnknownSnapshot(TreeId), + /// A blob or a tree was fetched as a record. + #[error("{0} is not a record: fetch snapshots by walking them")] + NotARecord(ObjectId), /// A target was proposed for landing without a target to land on. #[error("landing a target needs a target to land on")] NoTarget, @@ -650,6 +856,22 @@ pub enum ServiceError { /// The kind it must be. kind: ObjectKind, }, + /// A push expected a change's heads to be where they no longer are. + #[error( + "change {} moved on the remote since it was fetched: run `origin fetch`, then \ + `origin converge {}` if it diverged", + change.short(), + change.short() + )] + Moved { + /// The change. + change: ChangeId, + /// Its heads now. + heads: Vec, + }, + /// An events cursor is of another op log than the repository's. + #[error("the cursor is of another op log, generation {0}: read from the end")] + OtherLog(String), /// A statement put isn't one the repository accepts: a promotion that /// doesn't match its original. #[error("statement {} cannot be put", .id.short())] @@ -675,10 +897,12 @@ impl ServiceError { #[must_use] pub fn code(&self) -> ErrorCode { match self { - Self::UnknownRepository(_) | Self::UnknownChange(_) | Self::UnknownTarget(_) => { - ErrorCode::NotFound - } + Self::UnknownRepository(_) + | Self::UnknownChange(_) + | Self::UnknownTarget(_) + | Self::UnknownSnapshot(_) => ErrorCode::NotFound, Self::NoTarget + | Self::NotARecord(_) | Self::Malformed { .. } | Self::Acceptance(_) | Self::Missing { .. } @@ -687,6 +911,8 @@ impl ServiceError { AnnotateError::Store(error) => Self::store_code(error), _ => ErrorCode::BadRequest, }, + Self::Moved { .. } => ErrorCode::Conflict, + Self::OtherLog(_) => ErrorCode::Expired, Self::Land(error) => match error { LandError::UnknownTarget(_) | LandError::UnknownChange(_) => ErrorCode::NotFound, LandError::TargetExists(_) @@ -695,6 +921,7 @@ impl ServiceError { | LandError::NotAuthority { .. } | LandError::WrongKey(_) | LandError::Divergent { .. } + | LandError::Moved(_) | LandError::Abandoned(_) | LandError::Unresolved { .. } | LandError::InvalidPolicy(_) => ErrorCode::Conflict, diff --git a/apps/origin-src/src/store/mod.rs b/apps/origin-src/src/store/mod.rs index 523e349f..b72b7276 100644 --- a/apps/origin-src/src/store/mod.rs +++ b/apps/origin-src/src/store/mod.rs @@ -247,9 +247,7 @@ impl Store

{ /// failure may not be durable. pub fn write_all(&self, objects: &[EncodedObject]) -> Result { let batch = self.batch(); - let written = Self::on_every_core(objects, |chunk| batch.write_all(chunk)) - .into_iter() - .sum::>()?; + let written = batch.stage_all(objects)?; batch.commit()?; Ok(written) } @@ -327,6 +325,24 @@ impl Store

{ Ok(self.path(id).exists() || self.find_packed(id)?.is_some()) } + /// Those of `ids` the store has, in order. + /// + /// # Errors + /// + /// Fails like [`Store::contains`]. + pub fn present + Copy>( + &self, + ids: impl IntoIterator, + ) -> Result, StoreError> { + let mut present = Vec::new(); + for id in ids { + if self.contains(id.into())? { + present.push(id); + } + } + Ok(present) + } + /// When this node stored the object: when its loose file was written, /// which packing it keeps. /// @@ -362,6 +378,29 @@ impl Store

{ Ok(LockedFile::open(&self.platform, &path, LockMode::Shared)?) } + /// The file whose lock makes checking a change's heads and publishing + /// revisions that move them one step. + const PUSH_LOCK: &str = "push"; + + /// Waits for the publication of revisions in progress, and keeps + /// others waiting while it's held: every put of revisions takes it, + /// checking the heads a push expects first (see + /// [`OriginService::push`](crate::service::OriginService::push)), and + /// the landing queue, checking that the head it rebases still is the + /// change's only one. + /// + /// # Errors + /// + /// Fails if the lock file cannot be opened. + pub fn lock_pushes(&self) -> Result { + let path = self.root.join(Self::PUSH_LOCK); + Ok(LockedFile::open( + &self.platform, + &path, + LockMode::Exclusive, + )?) + } + /// Waits for the writers holding objects, and keeps new ones waiting /// while it's held: garbage collection holds it while it finds what /// nothing refers to, and deletes it. @@ -480,6 +519,61 @@ impl Store

{ self.try_read(id)?.ok_or(StoreError::NotFound(id)) } + /// Reads objects the store must have, in order, from every core. + /// + /// # Errors + /// + /// Fails like [`Store::read`]. + pub fn read_all(&self, ids: &[ObjectId]) -> Result, StoreError> { + let chunks = Self::on_every_core(ids, |chunk| { + chunk + .iter() + .map(|&id| self.read(id)) + .collect::, _>>() + }); + let mut objects = Vec::with_capacity(ids.len()); + for chunk in chunks { + objects.extend(chunk?); + } + Ok(objects) + } + + /// Reads the objects `ids` names, in order, handing each to `each` until + /// it returns `false`. They're read from every core a chunk at a time, + /// chunks growing while their objects are small and shrinking while + /// they're large, so a chunk holds a few MiB. + /// + /// # Errors + /// + /// Fails like [`Store::read`], or as `each` does. + pub fn read_each>( + &self, + ids: &[ObjectId], + mut each: impl FnMut(EncodedObject) -> Result, + ) -> Result<(), E> { + /// The bytes a chunk aims for. + const TARGET: usize = 8 << 20; + let mut size = 16; + let mut rest = ids; + while !rest.is_empty() { + let (chunk, after) = rest.split_at(size.min(rest.len())); + rest = after; + let objects = self.read_all(chunk)?; + let bytes: usize = objects.iter().map(|object| object.bytes().len()).sum(); + if bytes < TARGET / 2 { + size = (size * 2).min(4096); + } else if bytes > TARGET * 2 { + size = (size / 2).max(1); + } + for object in objects { + if !each(object)? { + return Ok(()); + } + } + } + Ok(()) + } + /// Reads and decodes the record `id` names. /// /// # Errors @@ -597,11 +691,26 @@ impl Batch<'_, P> { Ok(self.store.packs.find(id).is_some()) } - /// Stages each of `objects` in turn, returning how many were new. - fn write_all(&self, objects: &[EncodedObject]) -> Result { - objects.iter().try_fold(0, |written, object| { - Ok(written + usize::from(self.write(object)?)) + /// Makes `path`, a file or a directory written in place, durable before + /// anything the batch stages is published. + pub(crate) fn sync_first(&self, path: impl Into) { + self.write.sync_with(path); + } + + /// Stages `objects` from every core, returning how many were new, as + /// [`Store::write_all`] does. + /// + /// # Errors + /// + /// Fails like [`Batch::write`]. + pub fn stage_all(&self, objects: &[EncodedObject]) -> Result { + Store::

::on_every_core(objects, |chunk| { + chunk.iter().try_fold(0, |written, object| { + Ok(written + usize::from(self.write(object)?)) + }) }) + .into_iter() + .sum() } /// Publishes every object staged, durably, and logs the new records in @@ -695,6 +804,9 @@ pub enum StoreError { /// Why decoding failed. source: DecodeError, }, + /// The operating system has no randomness to give. + #[error("cannot draw a random id")] + Random(#[source] getrandom::Error), /// The op log no longer keeps the entries after a cursor: retention /// deleted them. #[error("the op log no longer keeps the entries after #{after}: its oldest is #{oldest}")] diff --git a/apps/origin-src/src/sync.rs b/apps/origin-src/src/sync.rs new file mode 100644 index 00000000..5323d013 --- /dev/null +++ b/apps/origin-src/src/sync.rs @@ -0,0 +1,2296 @@ +//! Sync: sharing records with a [remote](crate::remote). +//! +//! - **Fetching** brings the chain of each target this node follows, those +//! it holds and those asked for, from the last acceptance it holds on. +//! Each acceptance must follow the one before it, and is verified against +//! the key of the authority named by the policy governing it +//! ([`Policy::governing`]): for an acceptance of `policy`, the one before +//! it, and for any other, the policy it names, which must be on the chain +//! of `policy` as this node verified it. Only the first acceptance of +//! `policy` is checked against its own. Acceptances this node merely +//! stores, such as the base of a revision fetched, never govern anything. +//! Each acceptance comes with what it landed, and the records of the +//! changes that brings. +//! - Then the records of each change that moved are compared with the +//! remote's, by id, and those missing fetched, with the records they +//! refer to, and the records of the changes those belong to. The changes +//! that moved are those the remote's events since the last fetch name +//! ([`Sync::moved`]), so a fetch that brings nothing costs the same +//! however long the history: only without a cursor, or once it expired, +//! are all the changes this node knows compared. +//! - Snapshots don't come with records: [`Sync::need`] fetches one when +//! something reads it, such as a checkout, so a clone fetches one +//! snapshot, its target's tip. A snapshot is marked while it's fetched +//! ([`Store::fetching`]), so one whose fetch was interrupted is fetched +//! again rather than taken for whole. +//! - **Pushing** shares revisions, with their snapshots, and statements. It +//! never lands anything. It first compares each change's heads with the +//! remote's, and refuses before uploading anything when the remote has a +//! revision this node lacks: the change is stale. The remote checks the +//! heads again as it publishes the revisions, so a push racing another +//! is refused too. Revisions only go to a remote that has the acceptance +//! they're based on. Each object goes after those it needs: snapshots +//! and attachments, then revisions, each with its reserved statements, +//! then the other statements. A statement that +//! needs one the remote neither has nor gets stays here, and the push +//! says so. +//! +//! Transfers are bounded, in both directions: a push is split into +//! requests of at most [`Pack::LIMIT`], and a fetch's answers stop there, +//! with the rest asked for again (see [`Part`]). Each request of a push +//! expects the heads the requests before it left. A revision goes in one +//! request with its reserved statements, such as its conflicts and its +//! abandonment, and the other statements about one record go in one +//! request, so the remote never lands a revision whose abandonment, or a +//! result whose replacement, is still to come. Fetched records are +//! written before the chains that refer to them. + +use std::collections::hash_map::Entry; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::{iter, mem}; + +use origin_platform::Native; +use origin_shared::api::{ + ErrorCode, EventsQuery, Expected, FetchRecords, FetchSnapshot, Inventory, InventoryRequest, + List, LogQuery, PutQuery, +}; +use origin_shared::event::EventKind; +use origin_shared::object::{BadSignature, DecodeError}; +use origin_shared::pack::{Pack, Packer, Part}; +use origin_shared::{ + Acceptance, AcceptanceId, Base, ChangeId, EncodedObject, Head, ObjectId, ObjectKind, Principal, + PrincipalName, Record, RecordId, Revision, RevisionId, Source, Statement, StatementId, + TargetName, TreeId, +}; + +use crate::annotations::Annotations; +use crate::index::Index; +use crate::policy::{Policy, PolicyError}; +use crate::remote::{Remote, RemoteError}; +use crate::service::Dependencies; +use crate::store::{Batch, Store, StoreError}; + +/// A store, and the remote it syncs with. +pub struct Sync<'a> { + store: &'a Store, + remote: &'a Remote, + /// This node, which flags the divergence fetched revisions reveal. + node: Principal, + /// The most bytes a push's request holds: [`Pack::LIMIT`], but in + /// tests. + pack_limit: usize, +} + +/// A target whose chain a fetch extended. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Advanced { + /// The target. + pub target: TargetName, + /// Its head before, if this node followed it. + pub from: Option, + /// Its head now. + pub to: Head, +} + +/// What fetching records did. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Fetched { + /// The revisions fetched. + pub revisions: usize, + /// The statements fetched. + pub statements: usize, +} + +/// What a fetch did. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Fetch { + /// The targets whose chains it extended. + pub advanced: Vec, + /// The records it fetched. + pub fetched: Fetched, + /// The remote's event cursor to fetch from next time. + pub cursor: Cursor, +} + +/// Where the remote's events are read from: after a sequence number of +/// its op log, which only means something in the log of its generation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Cursor { + /// The generation of the remote's op log. + pub generation: String, + /// The sequence number to read after. + pub next: u64, +} + +/// What the remote's events say moved since a cursor. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Moved { + /// The remote's cursor after those events: where the next fetch reads + /// from. + pub cursor: Cursor, + /// The changes that have revisions or statements the events name, or + /// `None` if the events since the cursor are unknown: there was none, + /// the remote no longer keeps them, or it's of another op log. + pub changes: Option>, +} + +/// What a push shared. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Pushed { + /// The revisions the remote didn't have. + pub revisions: usize, + /// The statements the remote didn't have. + pub statements: usize, + /// Every object sent: those, and the blobs and trees of their + /// snapshots the remote lacked. + pub objects: usize, + /// The statements that stay here: they need a record the remote + /// neither has nor got, such as a statement they supersede about + /// another change, or the original they promote. + pub unsent: Vec, +} + +/// The statements a push sends, in order, and those that stay. +struct Statements { + statements: Vec<(StatementId, Statement)>, + unsent: Vec, +} + +/// What a push sends, and what each of its requests expects of the +/// remote. +#[derive(Default)] +struct Transfer { + /// The objects, each after those it needs. + ids: Vec, + /// Those that go in the same request as the one before them. + joined: HashSet, + /// The change of each revision sent, and those it evolves from. + revisions: HashMap)>, + /// The heads of each change on the remote, as of the requests sent. + heads: HashMap>, +} + +/// The new acceptances of a target, verified and about to be appended. +struct Chain { + target: TargetName, + /// This node's head of the target, if it has one. + from: Option, + /// The remote's head: the last of `acceptances`. + to: Head, + acceptances: Vec<(AcceptanceId, Acceptance)>, +} + +/// Items, by position, joined into groups: each group is named by its +/// first item. +struct Groups(Vec); + +impl Groups { + /// `len` items, each in a group of its own. + fn new(len: usize) -> Self { + Self((0..len).collect()) + } + + /// The first item of the group of the one `at`. + fn root(&mut self, mut at: usize) -> usize { + while self.0[at] != at { + self.0[at] = self.0[self.0[at]]; + at = self.0[at]; + } + at + } + + /// Joins the groups of `a` and `b`. + fn join(&mut self, a: usize, b: usize) { + let (a, b) = (self.root(a), self.root(b)); + self.0[a.max(b)] = a.min(b); + } +} + +impl Transfer { + /// Adds `revisions`, each in one request with the reserved statements + /// of `statements` about it, such as its conflicts or its abandonment, + /// and the statements those need: so the remote never has a revision + /// without them. Those go after the last revision they need, in one + /// request with every revision from the first they're about. Returns + /// the other statements. + fn add_revisions( + &mut self, + revisions: &[(RevisionId, Revision)], + statements: Vec<(StatementId, Statement)>, + ) -> Vec<(StatementId, Statement)> { + let sent: HashMap = revisions + .iter() + .enumerate() + .map(|(at, &(id, _))| (RecordId::Revision(id), at)) + .collect(); + let position = Self::positions(&statements); + // Those that go with a revision, grouped with those they need, + // which come before them. + let mut with: Vec = statements + .iter() + .map(|(_, statement)| { + statement.kind.is_reserved() && sent.contains_key(&statement.subject) + }) + .collect(); + let mut groups = Groups::new(statements.len()); + for at in (0..statements.len()).rev() { + if !with[at] { + continue; + } + for (needed, _) in Dependencies::of_statement(&statements[at].1).records { + if let Some(&other) = position.get(&needed) { + with[other] = true; + groups.join(at, other); + } + } + } + // Each group's revisions: the first a reserved one is about, and + // the last any needs. + let mut spans: HashMap = HashMap::new(); + for (at, (_, statement)) in statements.iter().enumerate() { + if let Some(&revision) = sent.get(&statement.subject) + && with[at] + { + let span = spans.entry(groups.root(at)).or_insert((usize::MAX, 0)); + if statement.kind.is_reserved() { + span.0 = span.0.min(revision); + } + span.1 = span.1.max(revision); + } + } + let mut joined = vec![false; revisions.len()]; + let mut after = vec![Vec::new(); revisions.len()]; + for &(first, last) in spans.values() { + joined[first + 1..=last].fill(true); + } + for at in (0..statements.len()).filter(|&at| with[at]) { + after[spans[&groups.root(at)].1].push(statements[at].0); + } + for (((id, revision), joined), after) in revisions.iter().zip(joined).zip(after) { + if joined { + self.joined.insert((*id).into()); + } + self.ids.push((*id).into()); + self.revisions.insert( + (*id).into(), + (revision.change, revision.predecessors.clone()), + ); + for statement in after { + self.joined.insert(statement.into()); + self.ids.push(statement.into()); + } + } + statements + .into_iter() + .zip(with) + .filter_map(|(statement, with)| (!with).then_some(statement)) + .collect() + } + + /// Adds `statements`, each after those of them it needs, so that those + /// about one record, with those they need or that need them, go in + /// one request: then a landing waiting on the record never takes a + /// result whose replacement, or the change's abandonment, is still to + /// come. + fn add_statements(&mut self, statements: &[(StatementId, Statement)]) { + let position = Self::positions(statements); + let mut groups = Groups::new(statements.len()); + let mut about: HashMap = HashMap::new(); + for (at, (_, statement)) in statements.iter().enumerate() { + let same_subject = *about.entry(statement.subject).or_insert(at); + let needed = Dependencies::of_statement(statement) + .records + .into_iter() + .filter_map(|(id, _)| position.get(&id).copied()); + for other in iter::once(same_subject).chain(needed) { + groups.join(at, other); + } + } + let mut members: BTreeMap> = BTreeMap::new(); + for at in 0..statements.len() { + members.entry(groups.root(at)).or_default().push(at); + } + for members in members.into_values() { + for (nth, at) in members.into_iter().enumerate() { + let id = statements[at].0.into(); + if nth > 0 { + self.joined.insert(id); + } + self.ids.push(id); + } + } + } + + /// Where each of `statements` is among them. + fn positions(statements: &[(StatementId, Statement)]) -> HashMap { + statements + .iter() + .enumerate() + .map(|(at, &(id, _))| (id.into(), at)) + .collect() + } + + /// The heads the changes of `revisions`, which a request sends, must + /// have on the remote before it: those after the requests before. + /// Then moves them, as the request does. + fn expect(&mut self, revisions: &[ObjectId]) -> Vec { + let mut expect: Vec = Vec::new(); + for id in revisions { + let (change, predecessors) = &self.revisions[id]; + let heads = self.heads.entry(*change).or_default(); + if !expect.iter().any(|expected| expected.change == *change) { + expect.push(Expected { + change: *change, + heads: heads.clone(), + }); + } + heads.retain(|head| !predecessors.contains(head)); + heads.push((*id).into()); + } + expect + } +} + +impl<'a> Sync<'a> { + /// The most events a page of [`Sync::moved`] asks for: as many as a + /// server sends. + const EVENTS: u32 = 1024; + + /// Syncs `store` with `remote`, as the node `node`. + pub fn new(store: &'a Store, remote: &'a Remote, node: PrincipalName) -> Self { + Self { + store, + remote, + node: Principal::Node(node), + pack_limit: Pack::LIMIT, + } + } + + /// Syncs pushing requests of at most `limit` bytes. + #[cfg(test)] + fn with_pack_limit(self, limit: usize) -> Self { + Self { + pack_limit: limit, + ..self + } + } + + /// Fetches as the module explains: the chains of the targets this node + /// follows and of `targets`, then the records of `changes` and of those + /// this node knows that moved since `cursor`, the remote's event cursor + /// as of the last fetch, or of all those it knows without one. + /// + /// # Errors + /// + /// Fails like [`Sync::fetch_chains`] and [`Sync::fetch_changes`]. + pub fn fetch( + &self, + targets: &[TargetName], + changes: &[ChangeId], + cursor: Option<&Cursor>, + ) -> Result { + // Read first, so what moves while this fetches is fetched again. + let moved = self.moved(cursor)?; + let (advanced, landed) = self.chains(targets)?; + let mut wanted: Vec = match &moved.changes { + // Nothing moved: the index needn't even be read. + Some(moved) if moved.is_empty() => Vec::new(), + Some(moved) => { + let index = Index::open(self.store)?; + moved + .iter() + .copied() + .filter(|&change| !index.revisions(change).is_empty()) + .collect() + } + None => Index::open(self.store)?.changes().collect(), + }; + wanted.extend(changes.iter().chain(&landed)); + wanted.sort_unstable(); + wanted.dedup(); + Ok(Fetch { + advanced, + fetched: self.fetch_changes(&wanted)?, + cursor: moved.cursor, + }) + } + + /// Fetches the chains of `policy`, of the targets this node holds, and + /// of `more`, with what they landed, as the module explains. Returns + /// the targets that moved, `policy` first. Learns the names of all the + /// remote's targets, so acceptances of those this node doesn't follow + /// can be described. + /// + /// # Errors + /// + /// Fails if the remote doesn't have a target of `more`, a target's + /// chain differs from the remote's, an acceptance doesn't verify, or + /// the remote or the store fails. + pub fn fetch_chains(&self, more: &[TargetName]) -> Result, SyncError> { + let (advanced, landed) = self.chains(more)?; + self.fetch_changes(&landed)?; + Ok(advanced) + } + + /// [`Sync::fetch_chains`], but for the records of the changes that + /// landed, which it returns, for the caller to fetch. + fn chains(&self, more: &[TargetName]) -> Result<(Vec, Vec), SyncError> { + let remote: HashMap = self + .remote + .targets()? + .into_iter() + .map(|target| (target.name, target.head)) + .collect(); + for name in remote.keys() { + self.store.name_target(name)?; + } + let mut follow: BTreeSet = self.store.targets()?.into_iter().collect(); + for target in more { + if !remote.contains_key(target) { + return Err(SyncError::UnknownTarget(target.clone())); + } + follow.insert(target.clone()); + } + follow.remove(&TargetName::POLICY); + let mut chains = Vec::new(); + for target in [TargetName::POLICY].into_iter().chain(follow) { + // Targets only this node has stay as they are. + if let Some(&head) = remote.get(&target) { + chains.extend(self.chain(target, head)?); + } + } + self.verify(&chains)?; + let appending: HashSet = chains + .iter() + .flat_map(|chain| &chain.acceptances) + .map(|&(id, _)| id.into()) + .collect(); + let landed = chains + .iter() + .flat_map(|chain| &chain.acceptances) + .flat_map(|(_, acceptance)| &acceptance.body().landed) + .map(|&source| match source { + Source::Revision(id) => id.into(), + Source::Target(id) => id.into(), + }); + // Objects found stored stay until the chains refer to them. + let _held = self.store.hold_objects()?; + let held = self.store.batch(); + let records = self.closure(&held, landed, &appending)?; + held.commit()?; + + let mut revised = Self::revised(&records)?; + revised.sort_unstable(); + revised.dedup(); + if !records.is_empty() { + self.store.write_all(&records)?; + } + let mut advanced = Vec::new(); + for chain in chains { + let acceptances: Vec<_> = chain.acceptances.into_iter().map(|(_, a)| a).collect(); + self.store.append_all(&chain.target, &acceptances, &[])?; + advanced.push(Advanced { + target: chain.target, + from: chain.from, + to: chain.to, + }); + } + // The changes that landed come whole, so the next fetch only looks + // for what moved since. + Ok((advanced, revised)) + } + + /// The acceptances of `target` after the last this node holds, up to + /// the remote's head, `theirs`, each checked against its id and target, + /// and to follow the one before it. `None` if there are none. + fn chain(&self, target: TargetName, theirs: Head) -> Result, SyncError> { + let ours = self.store.head(&target)?; + if ours == Some(theirs) { + return Ok(None); + } + let query = LogQuery { + target: target.clone(), + since: ours.map_or(0, |ours| ours.seq), + limit: None, + }; + let mut entries = self.remote.log(&query)?.into_iter(); + // The remote's log goes through this node's head. + if let Some(ours) = ours + && entries + .next() + .is_none_or(|entry| entry.id != ours.acceptance) + { + return Err(SyncError::Diverged(target)); + } + let mut acceptances = Vec::new(); + let mut before = ours; + for entry in entries { + let body = entry.acceptance.body(); + let unverified = |why| SyncError::Unverified { + target: target.clone(), + seq: body.seq, + why, + }; + if entry.acceptance.encode().id() != entry.id.into() || body.target != target.id() { + return Err(unverified(Unverified::Id(entry.id))); + } + if body.seq != before.map_or(0, |before| before.seq + 1) + || body.prev != before.map(|before| before.acceptance) + { + return Err(unverified(Unverified::Order)); + } + before = Some(Head { + seq: body.seq, + acceptance: entry.id, + }); + acceptances.push((entry.id, entry.acceptance)); + } + let Some(to) = before.filter(|_| !acceptances.is_empty()) else { + return Ok(None); + }; + Ok(Some(Chain { + target, + from: ours, + to, + acceptances, + })) + } + + /// Checks that the authority each acceptance's governing policy names + /// signed it, as the module explains. `chains` start with `policy`'s, + /// whose acceptances may govern those after them once verified. The + /// policies' snapshots are fetched as needed, and those of `policy`'s + /// new acceptances always, so the policy in force can be read. + fn verify(&self, chains: &[Chain]) -> Result<(), SyncError> { + let mut policies: HashMap = HashMap::new(); + // The acceptances of `policy` verified so far, and their snapshots. + let mut verified: HashMap = HashMap::new(); + for chain in chains { + for (id, acceptance) in &chain.acceptances { + let body = acceptance.body(); + let unverified = |why| SyncError::Unverified { + target: chain.target.clone(), + seq: body.seq, + why, + }; + // Small, and the policy in force once it's the tip. + if chain.target == TargetName::POLICY { + self.need(body.state, &[])?; + } + let governing = Policy::governing(*id, body) + .map_err(|error| unverified(Unverified::Policy(error)))?; + let policy = match policies.entry(governing) { + Entry::Occupied(policy) => policy.into_mut(), + Entry::Vacant(vacant) => { + // Only the first acceptance of `policy` governs + // itself: `chain` has it follow nothing. + let state = if governing == *id { + body.state + } else if let Some(&state) = verified.get(&governing) { + state + } else { + self.on_policy_chain(governing)? + .ok_or_else(|| unverified(Unverified::OffChain(governing)))? + }; + self.need(state, &[])?; + let policy = Policy::of_snapshot(self.store, state)? + .ok_or(unverified(Unverified::NoPolicy))?; + vacant.insert(policy) + } + }; + let authority = policy + .authority(&chain.target) + .ok_or(unverified(Unverified::NoAuthority))?; + let key = policy + .node_key(authority) + .ok_or_else(|| unverified(Unverified::NoKey(authority.clone())))?; + acceptance + .verify(key) + .map_err(|error| unverified(Unverified::Signature(error)))?; + if chain.target == TargetName::POLICY { + verified.insert(*id, body.state); + } + } + } + Ok(()) + } + + /// The snapshot of `id`, if it's on this node's chain of `policy`, + /// which it verified as it appended it. + fn on_policy_chain(&self, id: AcceptanceId) -> Result, SyncError> { + let acceptance = match self.store.get(id) { + Ok(acceptance) => acceptance, + Err(StoreError::NotFound(_)) => return Ok(None), + Err(error) => return Err(error.into()), + }; + let body = acceptance.body(); + let on_chain = body.target == TargetName::POLICY.id() + && self.store.acceptance_at(body.target, body.seq)? == Some(id); + Ok(on_chain.then_some(body.state)) + } + + /// The remote's cursor after its events since `since`, and the changes + /// they say moved: those with a revision created or a statement issued + /// about them, or about a statement about them. Without `since`, or if + /// the remote no longer keeps its events, or its op log is of another + /// generation, as once it was restored, the remote's current cursor, + /// and no changes: then every change may have moved. + /// + /// # Errors + /// + /// Fails if the remote or the store fails. + pub fn moved(&self, since: Option<&Cursor>) -> Result { + // Should a name not parse, every event comes, and those of other + // kinds name no change. + let kinds = ["revision.created", "statement.issued"] + .into_iter() + .filter_map(|kind| kind.parse().ok()) + .collect(); + let mut query = EventsQuery { + kind: List(kinds), + limit: Some(Self::EVENTS), + ..EventsQuery::default() + }; + let everything = |query: &mut EventsQuery| { + query.cursor = None; + query.generation = None; + let page = self.remote.events(query)?; + Ok(Moved { + cursor: Cursor { + generation: page.generation, + next: page.next, + }, + changes: None, + }) + }; + let Some(since) = since else { + return everything(&mut query); + }; + let mut cursor = since.clone(); + let mut events = Vec::new(); + loop { + query.cursor = Some(cursor.next); + query.generation = Some(cursor.generation.clone()); + let page = match self.remote.events(&query) { + Err(error) if error.code() == Some(ErrorCode::Expired) => { + return everything(&mut query); + } + page => page?, + }; + cursor.next = page.next; + // A page short of the limit holds every event there was. + let last = page.events.len() < Self::EVENTS as usize; + events.extend(page.events.into_iter().map(|event| event.kind)); + if last { + break; + } + } + Ok(Moved { + cursor, + changes: Some(self.changes_moved(&events)?), + }) + } + + /// The changes `events` name, as [`Sync::moved`] says. A statement is + /// traced to its subject through the events, then this node's records: + /// one about a record it knows nothing of is about no change it knows. + fn changes_moved(&self, events: &[EventKind]) -> Result, SyncError> { + let mut changes = HashSet::new(); + let mut revisions = HashMap::new(); + let mut subjects = HashMap::new(); + for event in events { + match *event { + EventKind::RevisionCreated { + revision, change, .. + } => { + changes.insert(change); + revisions.insert(revision, change); + } + EventKind::StatementIssued { + statement, subject, .. + } => { + subjects.insert(statement, subject); + } + _ => {} + } + } + for &subject in subjects.values() { + let mut subject = subject; + // A chain of statements ends at a record, or one unknown. + let change = loop { + match subject { + RecordId::Revision(id) => match revisions.get(&id) { + Some(&change) => break Some(change), + None => break self.stored(id)?.map(|revision| revision.change), + }, + RecordId::Statement(id) => match subjects.get(&id) { + Some(&next) if next != subject => subject = next, + _ => match self.stored(id)? { + Some(statement) => subject = statement.subject, + None => break None, + }, + }, + RecordId::Acceptance(_) | RecordId::Snapshot(_) => break None, + } + }; + changes.extend(change); + } + Ok(changes) + } + + /// The record `id` names, if this node has it. + fn stored(&self, id: I) -> Result, StoreError> { + match self.store.get(id) { + Ok(record) => Ok(Some(record)), + Err(StoreError::NotFound(_)) => Ok(None), + Err(error) => Err(error), + } + } + + /// Fetches the records of `changes` the remote has and this node + /// doesn't: their revisions and the statements about them, with the + /// records they refer to, and then the records of the changes the + /// revisions fetched belong to, so every change this node learns of + /// comes whole. Flags the changes that diverged. + /// + /// # Errors + /// + /// Fails if the remote or the store fails. + pub fn fetch_changes(&self, changes: &[ChangeId]) -> Result { + let mut fetched = Fetched::default(); + let mut asked: HashSet = changes.iter().copied().collect(); + let mut next: Vec = asked.iter().copied().collect(); + let mut revised = BTreeSet::new(); + // Objects found stored stay until what refers to them is written. + let _held = self.store.hold_objects()?; + while !next.is_empty() { + let inventories = self.remote.inventory(&InventoryRequest { + changes: mem::take(&mut next), + })?; + let ids = inventories.iter().flat_map(|inventory| { + let revisions = inventory.revisions.iter().map(|&id| ObjectId::from(id)); + revisions.chain(inventory.statements.iter().map(|&id| id.into())) + }); + let held = self.store.batch(); + let records = self.closure(&held, ids, &HashSet::new())?; + held.commit()?; + for change in Self::revised(&records)? { + fetched.revisions += 1; + revised.insert(change); + if asked.insert(change) { + next.push(change); + } + } + fetched.statements += records + .iter() + .filter(|record| record.kind().ok() == Some(ObjectKind::Statement)) + .count(); + if !records.is_empty() { + self.store.write_all(&records)?; + } + } + if !revised.is_empty() { + let annotations = Annotations::open(self.store)?; + for change in revised { + annotations.flag_divergence(change, self.node.clone())?; + } + } + Ok(fetched) + } + + /// The change of each revision of `records`, one for each. + fn revised(records: &[EncodedObject]) -> Result, SyncError> { + let mut changes = Vec::new(); + for record in records { + if record.kind().ok() == Some(ObjectKind::Revision) { + let revision: Revision = + record.decode_as().map_err(|source| SyncError::Malformed { + id: record.id(), + source, + })?; + changes.push(revision.change); + } + } + Ok(changes) + } + + /// Fetches the records `ids` names that this node lacks, other than + /// those of `skip`, with the records they refer to, recursively: a + /// revision's base and dependencies, a statement's subject and the + /// statements it supersedes or promotes, but not their snapshots. Each + /// comes with the blob it carries. Those this node has are made durable + /// by `held`, which must commit before anything refers to them. + fn closure( + &self, + held: &Batch<'_, Native>, + ids: impl IntoIterator, + skip: &HashSet, + ) -> Result, SyncError> { + let mut seen = HashSet::new(); + let mut wanted = Vec::new(); + let mut want = |id: ObjectId, wanted: &mut Vec| { + if !skip.contains(&id) && !held.depend(id)? && seen.insert(id) { + wanted.push(id); + } + Ok::<_, StoreError>(()) + }; + for id in ids { + want(id, &mut wanted)?; + } + let mut objects = Vec::new(); + while !wanted.is_empty() { + let mut request = FetchRecords { + ids: mem::take(&mut wanted), + }; + while !request.ids.is_empty() { + let Part { pack, more } = self.remote.records(&request)?; + // A part that stops short holds the records up to the last + // it has: the rest come from asking again. + let rest = if more { + let position: HashMap = request + .ids + .iter() + .enumerate() + .map(|(position, &id)| (id, position)) + .collect(); + let last = pack + .objects + .iter() + .filter_map(|object| position.get(&object.id())) + .max(); + last.map_or_else(Vec::new, |&last| request.ids.split_off(last + 1)) + } else { + Vec::new() + }; + for object in pack.objects { + if matches!( + object.kind(), + Ok(ObjectKind::Revision | ObjectKind::Statement) + ) { + let decoded = object.decode().map_err(|source| SyncError::Malformed { + id: object.id(), + source, + })?; + for (id, _) in Dependencies::of(&decoded).records { + want(id, &mut wanted)?; + } + } + objects.push(object); + } + request.ids = rest; + } + } + Ok(objects) + } + + /// Fetches the trees and blobs of `snapshot`, unless the store holds it + /// whole, skipping what the snapshots `have`, which it holds whole, + /// hold at the same path. Returns whether it fetched. + /// + /// # Errors + /// + /// Fails if the remote doesn't have the snapshot, or the remote or the + /// store fails. + pub fn need(&self, snapshot: TreeId, have: &[TreeId]) -> Result { + if self.store.holds(snapshot)? { + return Ok(false); + } + let request = FetchSnapshot { + snapshot, + have: self.store.held_whole(have.iter().copied())?, + paths: Vec::new(), + skip: 0, + }; + self.fetch_parts(request, Some(snapshot))?; + Ok(true) + } + + /// Fetches `snapshot` whole again, whatever the store holds: what the + /// store lacks of a snapshot it has the root of, which reading it + /// found. + /// + /// # Errors + /// + /// Fails like [`Sync::need`]. + pub fn repair(&self, snapshot: TreeId) -> Result<(), SyncError> { + let request = FetchSnapshot { + snapshot, + have: Vec::new(), + paths: Vec::new(), + skip: 0, + }; + self.fetch_parts(request, Some(snapshot)) + } + + /// Fetches what `request` asks for, part after part, in one batch. With + /// `whole`, the snapshot it fetches whole, which is marked until the + /// batch is durable. + fn fetch_parts( + &self, + mut request: FetchSnapshot, + whole: Option, + ) -> Result<(), SyncError> { + // Objects found stored stay until the trees referring to them are. + let _held = self.store.hold_objects()?; + let batch = self.store.batch(); + let fetching = whole + .map(|snapshot| self.store.fetching(snapshot, &batch)) + .transpose()?; + loop { + let Part { pack, more } = self.remote.snapshot(&request)?; + batch.stage_all(&pack.objects)?; + if !more || pack.objects.is_empty() { + break; + } + request.skip += pack.objects.len() as u64; + } + batch.commit()?; + if let Some(fetching) = fetching { + fetching.done()?; + } + Ok(()) + } + + /// Shares `changes`, and the changes below them, with the remote, as + /// the module explains. + /// + /// # Errors + /// + /// Fails if a change has no revision here, is stale, or is based on an + /// acceptance the remote doesn't have, or if the remote or the store + /// fails. Nothing is sent then, unless the change moved on the remote + /// while the push was sending: then the snapshots and records sent + /// before stay there. + pub fn push(&self, changes: &[ChangeId]) -> Result { + let index = Index::open(self.store)?; + let changes = self.stack(&index, changes)?; + let inventories = self.inventories(&index, &changes)?; + let mut shared: HashSet = HashSet::new(); + for inventory in &inventories { + shared.extend(inventory.revisions.iter().map(|&id| ObjectId::from(id))); + shared.extend(inventory.statements.iter().map(|&id| ObjectId::from(id))); + } + + let mut revisions = Vec::new(); + for &change in &changes { + for &id in index.revisions(change) { + if !shared.contains(&id.into()) { + revisions.push((id, self.store.get(id)?)); + } + } + } + let revisions = Self::bottom_up(revisions); + let mut checked = HashSet::new(); + for (id, revision) in &revisions { + if let Base::Acceptance(base) = revision.base + && checked.insert(base) + { + self.check_shared(*id, revision, base)?; + } + } + + // Snapshots and attachments first, then revisions, then statements. + let mut content = Vec::new(); + let mut seen = HashSet::new(); + for (id, revision) in &revisions { + // The remote has these snapshots, or gets them first. + let mut have = vec![match revision.base { + Base::Acceptance(base) => self.store.get(base)?.body().state, + Base::Revision(base) => self.store.get(base)?.snapshot, + }]; + for &predecessor in &revision.predecessors { + if shared.contains(&predecessor.into()) { + have.push(self.store.get(predecessor)?.snapshot); + } + } + let have = self.store.held_whole(have)?; + // Children first: a tree goes after what it holds. + for placed in self + .store + .added(revision.snapshot, &have, &[])? + .into_iter() + .rev() + { + if seen.insert(placed.entry.id) { + content.push(placed.entry.id); + } + } + if seen.insert(revision.description.into()) { + content.push(revision.description.into()); + } + shared.insert((*id).into()); + } + let Statements { statements, unsent } = self.statements(&index, &changes, &mut shared)?; + for (_, statement) in &statements { + if let Some(payload) = statement.payload + && self.store.contains(payload.into())? + && seen.insert(payload.into()) + { + content.push(payload.into()); + } + } + let objects = content.len() + revisions.len() + statements.len(); + let mut transfer = Transfer { + ids: content, + heads: inventories + .into_iter() + .map(|inventory| (inventory.change, inventory.heads)) + .collect(), + ..Transfer::default() + }; + let statements = transfer.add_revisions(&revisions, statements); + transfer.add_statements(&statements); + if let Err(error) = self.send(transfer) { + // The remote refused as a change moved: say which. + if let SyncError::Remote(remote) = &error + && remote.code() == Some(ErrorCode::Conflict) + { + self.inventories(&Index::open(self.store)?, &changes)?; + } + return Err(error); + } + Ok(Pushed { + revisions: revisions.len(), + statements: statements.len(), + objects, + unsent, + }) + } + + /// What the remote holds of `changes`, refusing if it has a head of one + /// this node lacks: the change is stale. + fn inventories( + &self, + index: &Index, + changes: &[ChangeId], + ) -> Result, SyncError> { + let inventories = self.remote.inventory(&InventoryRequest { + changes: changes.to_vec(), + })?; + for inventory in &inventories { + let local = index.revisions(inventory.change); + if let Some(&head) = inventory.heads.iter().find(|head| !local.contains(head)) { + return Err(SyncError::Stale { + change: inventory.change, + head, + }); + } + } + Ok(inventories) + } + + /// `revisions`, each after those of them it evolves from, is based on + /// or depends on. + fn bottom_up(revisions: Vec<(RevisionId, Revision)>) -> Vec<(RevisionId, Revision)> { + let position: HashMap = revisions + .iter() + .enumerate() + .map(|(position, &(id, _))| (id, position)) + .collect(); + let mut placed = vec![false; revisions.len()]; + let mut order = Vec::with_capacity(revisions.len()); + for start in 0..revisions.len() { + // Depth first, placing each revision once those below it are. + let mut stack = vec![(start, false)]; + while let Some((at, below_placed)) = stack.pop() { + if placed[at] { + continue; + } + if below_placed { + placed[at] = true; + order.push(at); + continue; + } + stack.push((at, true)); + let revision = &revisions[at].1; + let base = match revision.base { + Base::Revision(base) => Some(base), + Base::Acceptance(_) => None, + }; + let below = base.iter().chain(&revision.deps); + for below in below.chain(&revision.predecessors) { + if let Some(&below) = position.get(below) + && !placed[below] + { + stack.push((below, false)); + } + } + } + } + let mut revisions: Vec<_> = revisions.into_iter().map(Some).collect(); + order + .into_iter() + .filter_map(|at| revisions[at].take()) + .collect() + } + + /// The statements about the revisions of `changes`, and about + /// statements about them, that the remote lacks, each after those of + /// them it needs, adding each to `shared`; and those that need a record + /// that isn't in `shared`, which stay. + fn statements( + &self, + index: &Index, + changes: &[ChangeId], + shared: &mut HashSet, + ) -> Result { + let mut subjects: Vec = changes + .iter() + .flat_map(|&change| index.revisions(change)) + .map(|&id| RecordId::Revision(id)) + .collect(); + let mut waiting = Vec::new(); + let mut seen = HashSet::new(); + while let Some(subject) = subjects.pop() { + for &id in index.statements(subject) { + if !seen.insert(id) { + continue; + } + subjects.push(RecordId::Statement(id)); + if shared.contains(&id.into()) { + continue; + } + waiting.push((id, self.store.get(id)?)); + } + } + let mut ordered = Vec::new(); + loop { + let before = waiting.len(); + waiting.retain(|(id, statement): &(StatementId, Statement)| { + let ready = Dependencies::of_statement(statement) + .records + .iter() + .all(|(needed, _)| shared.contains(needed)); + if ready { + shared.insert((*id).into()); + ordered.push((*id, statement.clone())); + } + !ready + }); + if waiting.len() == before { + break; + } + } + Ok(Statements { + statements: ordered, + unsent: waiting.into_iter().map(|(id, _)| id).collect(), + }) + } + + /// Puts the objects of `transfer`, in order, in requests of at most + /// [`Pack::LIMIT`], keeping together those it joins. + fn send(&self, mut transfer: Transfer) -> Result<(), SyncError> { + let ids = mem::take(&mut transfer.ids); + let mut packer = Packer::new(self.pack_limit); + let mut together = Vec::new(); + let mut filling = Vec::new(); + self.store.read_each(&ids, |object| { + if !transfer.joined.contains(&object.id()) { + let objects = mem::take(&mut together); + self.add(&mut packer, objects, &mut filling, &mut transfer)?; + } + together.push(object); + Ok::<_, SyncError>(true) + })?; + self.add(&mut packer, together, &mut filling, &mut transfer)?; + if !packer.is_empty() { + self.put(packer.finish(), &filling, &mut transfer)?; + } + Ok(()) + } + + /// Adds `together` to `packer`, all in one pack: if they don't fit, + /// first puts the pack it filled, whose revisions are `filling`. + fn add( + &self, + packer: &mut Packer, + together: Vec, + filling: &mut Vec, + transfer: &mut Transfer, + ) -> Result<(), SyncError> { + let revisions: Vec = together + .iter() + .map(EncodedObject::id) + .filter(|id| transfer.revisions.contains_key(id)) + .collect(); + if let Some(full) = packer.add_together(together) { + self.put(full, &mem::take(filling), transfer)?; + } + filling.extend(revisions); + Ok(()) + } + + /// Puts `pack`, whose revisions are `revisions`, if their changes have + /// the heads `transfer` expects of them. + fn put( + &self, + pack: Pack, + revisions: &[ObjectId], + transfer: &mut Transfer, + ) -> Result<(), SyncError> { + let query = PutQuery { + expect: List(transfer.expect(revisions)), + }; + self.remote.push(pack, &query)?; + Ok(()) + } + + /// `changes`, and every change below their revisions, following their + /// bases and dependencies. + fn stack(&self, index: &Index, changes: &[ChangeId]) -> Result, SyncError> { + let mut stack = Vec::new(); + let mut seen = HashSet::new(); + let mut next = changes.to_vec(); + while let Some(change) = next.pop() { + if !seen.insert(change) { + continue; + } + let revisions = index.revisions(change); + if revisions.is_empty() { + return Err(SyncError::UnknownChange(change)); + } + stack.push(change); + for &id in revisions { + let revision = self.store.get(id)?; + let base = match revision.base { + Base::Revision(base) => Some(base), + Base::Acceptance(_) => None, + }; + for below in base.into_iter().chain(revision.deps) { + if self.store.contains(below.into())? { + next.push(self.store.get(below)?.change); + } + } + } + } + Ok(stack) + } + + /// Checks that the remote has `base`, the acceptance the revision `id` + /// is based on. + fn check_shared( + &self, + id: RevisionId, + revision: &Revision, + base: AcceptanceId, + ) -> Result<(), SyncError> { + let body = self.store.get(base)?.body().clone(); + let unshared = |target| SyncError::Unshared { + change: revision.change, + revision: id, + target, + seq: body.seq, + }; + let Some(target) = self.store.target_name(body.target)? else { + return Err(unshared(None)); + }; + let query = LogQuery { + target: target.clone(), + since: body.seq, + limit: Some(1), + }; + match self.remote.log(&query) { + Ok(entries) if entries.first().is_some_and(|entry| entry.id == base) => Ok(()), + Ok(_) => Err(unshared(Some(target))), + Err(error) if error.code() == Some(ErrorCode::NotFound) => Err(unshared(Some(target))), + Err(error) => Err(error.into()), + } + } +} + +/// Why an acceptance fetched doesn't verify. +#[derive(Debug, thiserror::Error)] +pub enum Unverified { + /// It doesn't match its id, or belongs to another target. + #[error("it isn't acceptance {0} of the target")] + Id(AcceptanceId), + /// It doesn't follow the acceptance before it. + #[error("it doesn't follow the acceptance before it")] + Order, + /// It names no policy, or another than the one governing it. + #[error("it names the wrong policy")] + Policy(#[source] PolicyError), + /// The policy it names isn't on the chain of `policy`. + #[error("its policy {} isn't on the chain of `policy`", .0.short())] + OffChain(AcceptanceId), + /// Its policy has no `policy.toml`. + #[error("its policy has no policy.toml")] + NoPolicy, + /// Its policy names no authority for the target. + #[error("its policy names no authority for the target")] + NoAuthority, + /// Its policy lists no key for the authority. + #[error("its policy lists no key for node:{0}")] + NoKey(PrincipalName), + /// The authority didn't sign it. + #[error("its authority didn't sign it")] + Signature(#[source] BadSignature), +} + +/// A failure to sync with a remote. +#[derive(Debug, thiserror::Error)] +pub enum SyncError { + /// The remote has no such target. + #[error("the remote has no target `{0}`")] + UnknownTarget(TargetName), + /// The change has no revision to push. + #[error("change {} has no revisions", .0.short())] + UnknownChange(ChangeId), + /// A target's chain here isn't a start of the remote's. + #[error("the history of `{0}` here differs from the remote's")] + Diverged(TargetName), + /// An acceptance fetched doesn't verify. + #[error("acceptance #{seq} of `{target}` from the remote doesn't verify: {why}")] + Unverified { + /// Its target. + target: TargetName, + /// Its position. + seq: u64, + /// Why. + #[source] + why: Unverified, + }, + /// The remote has a revision of a change that this node lacks: the + /// change moved on there. + #[error( + "change {} is stale: the remote has revision {} of it, which this repository doesn't. \ + Run `origin fetch`, then `origin converge {}`", + change.short(), + head.short(), + change.short() + )] + Stale { + /// The change. + change: ChangeId, + /// The remote's head this node lacks. + head: RevisionId, + }, + /// A revision to push is based on an acceptance the remote doesn't + /// have. + #[error( + "revision {} of change {} is based on {} #{seq}, which the remote doesn't have: \ + start the change on a target the remote has, with `origin new --on`", + revision.short(), + change.short(), + target.as_ref().map_or_else(|| "a target".to_owned(), |target| format!("`{target}`")) + )] + Unshared { + /// The change. + change: ChangeId, + /// The revision. + revision: RevisionId, + /// The base's target, if this node knows its name. + target: Option, + /// The base's position. + seq: u64, + }, + /// The remote sent an object that doesn't decode. + #[error("the remote sent a malformed object {id}")] + Malformed { + /// The object. + id: ObjectId, + /// What is wrong. + source: DecodeError, + }, + /// The remote failed. + #[error(transparent)] + Remote(#[from] RemoteError), + /// A policy fetched cannot be read. + #[error(transparent)] + Policy(#[from] PolicyError), + /// Objects cannot be read or written. + #[error(transparent)] + Store(#[from] StoreError), +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use origin_platform::Native; + use origin_shared::event::Filter; + use origin_shared::{AcceptanceBody, Kind, NodeKey, RepoPath, Verdict}; + + use super::*; + use crate::events::Events; + use crate::queue::tests::{Repo, ci_test}; + use crate::service::{Node, OriginService}; + use crate::tree::Snapshots; + use crate::tree::tests::build; + + const MAIN_RULES: &str = "[targets.main]\nauthority = \"node:queue\"\n"; + + /// A node's own repository, empty, syncing with the queue's repository + /// of `server` in-process. + struct Client { + _dir: tempfile::TempDir, + store: Store, + remote: Remote, + } + + impl Client { + fn new(server: &Repo) -> Self { + Self::answered_in_parts(server, Pack::LIMIT) + } + + /// A client of a server answering in parts of `limit` bytes. + fn answered_in_parts(server: &Repo, limit: usize) -> Self { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let node = Node { + name: "queue".parse().unwrap(), + key: Arc::new(NodeKey::from_bytes(&[7; 32])), + }; + let service = OriginService::new(server.store.clone(), node).with_pack_limit(limit); + let remote = Remote::Local(Box::new(service)); + Self { + _dir: dir, + store, + remote, + } + } + + fn sync(&self) -> Sync<'_> { + Sync::new(&self.store, &self.remote, "laptop".parse().unwrap()) + } + + /// Writes a revision of `change` with `files` on `base`, with every + /// object it needs. + fn revision( + &self, + change: u8, + predecessors: &[RevisionId], + base: Base, + files: &[(&str, &str)], + ) -> RevisionId { + let mut objects: Vec<_> = files + .iter() + .map(|(_, content)| EncodedObject::blob(content.as_bytes())) + .collect(); + let description = EncodedObject::blob(b"work"); + let revision = Revision { + change: ChangeId::from_bytes([change; 16]), + predecessors: predecessors.to_vec(), + base, + deps: Vec::new(), + snapshot: build(&self.store, files), + description: description.id().into(), + author: "user:ada".parse().unwrap(), + } + .encode(); + objects.extend([description, revision.clone()]); + self.store.write_all(&objects).unwrap(); + revision.id().into() + } + } + + fn main_tip(store: &Store) -> Base { + Base::Acceptance(store.head(&TargetName::MAIN).unwrap().unwrap().acceptance) + } + + /// Writes a snapshot of `policy.toml` making `key` the key of + /// `node:queue`, the authority of `policy`, to `store`. + fn policy_snapshot(store: &Store, key: &NodeKey) -> TreeId { + let policy = format!( + "[nodes]\nqueue = \"{}\"\n[targets.policy]\nauthority = \"node:queue\"\n{MAIN_RULES}", + key.public() + ); + let (file, tree) = Policy::snapshot(&policy); + store.write_all(&[file, tree.clone()]).unwrap(); + tree.id().into() + } + + #[test] + fn fetches_verified_chains_and_what_they_landed_but_not_snapshots() { + let server = Repo::new(MAIN_RULES); + let landed = server.change_on_main(1, &[("a", "a")]); + let result = server + .annotations() + .issue(&ci_test(landed, "node:ci"), None) + .unwrap(); + server.land(1); + let client = Client::new(&server); + + let advanced = client.sync().fetch_chains(&[]).unwrap(); + assert_eq!(advanced.len(), 1, "`policy` is always followed"); + let advanced = client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + assert_eq!(advanced[0].to, server.tip(&TargetName::MAIN)); + assert_eq!( + client.store.log(&TargetName::MAIN).unwrap(), + server.store.log(&TargetName::MAIN).unwrap() + ); + assert!( + client.store.contains(landed.into()).unwrap(), + "what landed comes" + ); + assert!( + client.store.contains(result.into()).unwrap(), + "with the records of its change" + ); + assert!(Policy::in_force(&client.store).is_ok(), "the policy too"); + + let state = client + .store + .get(server.tip(&TargetName::MAIN).acceptance) + .unwrap() + .body() + .state; + assert!( + !client.store.contains(state.into()).unwrap(), + "snapshots are lazy" + ); + assert!(client.sync().need(state, &[]).unwrap()); + assert!(!client.sync().need(state, &[]).unwrap(), "once"); + assert_eq!(client.store.diff(state, state).unwrap(), []); + assert_eq!(client.sync().fetch_chains(&[]).unwrap(), []); + } + + #[test] + fn acceptances_that_do_not_verify_are_refused() { + let server = Repo::new(MAIN_RULES); + let tip = server.tip(&TargetName::MAIN); + let state = server.store.get(tip.acceptance).unwrap().body().state; + let forged = AcceptanceBody { + target: TargetName::MAIN.id(), + seq: 1, + prev: Some(tip.acceptance), + state, + landed: Vec::new(), + evidence: Vec::new(), + policy: Some(server.tip(&TargetName::POLICY).acceptance), + } + .sign(&NodeKey::from_bytes(&[8; 32])); + server.store.append(&TargetName::MAIN, &forged).unwrap(); + let client = Client::new(&server); + + let error = client.sync().fetch_chains(&[TargetName::MAIN]).unwrap_err(); + assert!( + matches!( + &error, + SyncError::Unverified { + seq: 1, + why: Unverified::Signature(_), + .. + } + ), + "{error}" + ); + assert_eq!( + client.store.head(&TargetName::MAIN).unwrap(), + None, + "nothing of the chain was appended" + ); + } + + #[test] + fn only_the_chain_of_policy_governs() { + let server = Repo::new(MAIN_RULES); + let client = Client::new(&server); + client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + // An acceptance of `policy` off its chain, signed by another key, + // which the client stores as the base of a revision it fetches. + let attacker = NodeKey::from_bytes(&[8; 32]); + let fake = AcceptanceBody { + target: TargetName::POLICY.id(), + seq: 40, + prev: None, + state: policy_snapshot(&server.store, &attacker), + landed: Vec::new(), + evidence: Vec::new(), + policy: None, + } + .sign(&attacker); + let fake_id: AcceptanceId = fake.encode().id().into(); + server.store.write_all(&[fake.encode()]).unwrap(); + server.revision(9, &[], Base::Acceptance(fake_id), &[("a", "a")]); + client + .sync() + .fetch_changes(&[ChangeId::from_bytes([9; 16])]) + .unwrap(); + assert!(client.store.contains(fake_id.into()).unwrap()); + + let tip = server.tip(&TargetName::MAIN); + let forged = AcceptanceBody { + target: TargetName::MAIN.id(), + seq: tip.seq + 1, + prev: Some(tip.acceptance), + state: server.store.get(tip.acceptance).unwrap().body().state, + landed: Vec::new(), + evidence: Vec::new(), + policy: Some(fake_id), + } + .sign(&attacker); + server.store.append(&TargetName::MAIN, &forged).unwrap(); + let error = client.sync().fetch_chains(&[]).unwrap_err(); + assert!( + matches!( + error, + SyncError::Unverified { why: Unverified::OffChain(id), .. } if id == fake_id + ), + "{error}" + ); + assert_eq!(client.store.head(&TargetName::MAIN).unwrap(), Some(tip)); + } + + #[test] + fn each_acceptance_of_policy_is_governed_by_the_one_before() { + // The policy replaces the authority's key, then the old key extends + // it, naming the policy it was replaced in, or the one before it. + let extended_by_the_old_key = |names_genesis: bool| { + let server = Repo::new(MAIN_RULES); + let client = Client::new(&server); + client.sync().fetch_chains(&[]).unwrap(); + let genesis = server.tip(&TargetName::POLICY).acceptance; + let old = NodeKey::from_bytes(&[7; 32]); + let rotation = AcceptanceBody { + target: TargetName::POLICY.id(), + seq: 1, + prev: Some(genesis), + state: policy_snapshot(&server.store, &NodeKey::from_bytes(&[8; 32])), + landed: Vec::new(), + evidence: Vec::new(), + policy: Some(genesis), + } + .sign(&old); + server.store.append(&TargetName::POLICY, &rotation).unwrap(); + client.sync().fetch_chains(&[]).unwrap(); + let rotation: AcceptanceId = rotation.encode().id().into(); + let reverted = AcceptanceBody { + target: TargetName::POLICY.id(), + seq: 2, + prev: Some(rotation), + state: server.store.get(genesis).unwrap().body().state, + landed: Vec::new(), + evidence: Vec::new(), + policy: Some(if names_genesis { genesis } else { rotation }), + } + .sign(&old); + server.store.append(&TargetName::POLICY, &reverted).unwrap(); + client.sync().fetch_chains(&[]).unwrap_err() + }; + let error = extended_by_the_old_key(true); + assert!( + matches!( + error, + SyncError::Unverified { + seq: 2, + why: Unverified::Policy(PolicyError::NotPredecessor(2)), + .. + } + ), + "{error}" + ); + let error = extended_by_the_old_key(false); + assert!( + matches!( + error, + SyncError::Unverified { + seq: 2, + why: Unverified::Signature(_), + .. + } + ), + "{error}" + ); + } + + #[test] + fn acceptances_fetched_before_their_chain_advance_it_once_appended() { + let server = Repo::new(MAIN_RULES); + server.change_on_main(1, &[("a", "a")]); + server.land(1); + let release = TargetName::new("release").unwrap(); + server + .queue() + .create_target(&release, &TargetName::MAIN) + .unwrap(); + let main_tip = server.tip(&TargetName::MAIN).acceptance; + let client = Client::new(&server); + let events = Events::new(&client.store); + + // `release` landed `main`'s tip, which comes as a record. + client.sync().fetch_chains(&[release]).unwrap(); + assert!(client.store.contains(main_tip.into()).unwrap()); + let advanced = |cursor| { + let page = events + .after(cursor, &Filter::default(), usize::MAX) + .unwrap(); + let main: Vec = page + .events + .iter() + .filter_map(|event| match &event.kind { + EventKind::TargetAdvanced { target, seq, .. } + if *target == TargetName::MAIN => + { + Some(*seq) + } + _ => None, + }) + .collect(); + (main, page.next) + }; + let (main, cursor) = advanced(0); + assert!(main.is_empty(), "not followed, so nothing advanced"); + + client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + assert_eq!( + advanced(cursor).0, + [0, 1], + "the acceptance held before advances `main` too, once appended" + ); + } + + #[test] + fn fetches_the_records_of_changes() { + let server = Repo::new(MAIN_RULES); + let first = server.change_on_main(1, &[("a", "a")]); + let annotations = server.annotations(); + annotations.issue(&ci_test(first, "node:ci"), None).unwrap(); + let client = Client::new(&server); + client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + + let fetched = client + .sync() + .fetch_changes(&[ChangeId::from_bytes([1; 16])]) + .unwrap(); + assert_eq!( + fetched, + Fetched { + revisions: 1, + statements: 1 + } + ); + assert!(client.store.contains(first.into()).unwrap()); + let again = client + .sync() + .fetch_changes(&[ChangeId::from_bytes([1; 16])]) + .unwrap(); + assert_eq!(again, Fetched::default()); + } + + #[test] + fn pushes_revisions_snapshots_and_statements() { + let server = Repo::new(MAIN_RULES); + let client = Client::new(&server); + client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + let base = main_tip(&client.store); + let first = client.revision(1, &[], base, &[("a", "1"), ("b", "1")]); + let second = client.revision(1, &[first], base, &[("a", "2"), ("b", "1")]); + let statement = ci_test(second, "user:ada").encode(); + client + .store + .write_all(std::slice::from_ref(&statement)) + .unwrap(); + + let change = ChangeId::from_bytes([1; 16]); + let pushed = client.sync().push(&[change]).unwrap(); + assert_eq!((pushed.revisions, pushed.statements), (2, 1), "{pushed:?}"); + assert_eq!( + pushed.objects, + 2 + 1 + 1 + 2 + 2, + "the revisions, their description, the statement, two trees and two blobs" + ); + for id in [first.into(), second.into(), statement.id()] { + assert!(server.store.contains(id).unwrap()); + } + assert_eq!(client.sync().push(&[change]).unwrap(), Pushed::default()); + } + + #[test] + fn pushes_statements_after_those_they_need() { + let server = Repo::new(MAIN_RULES); + let client = Client::new(&server); + client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + let base = main_tip(&client.store); + let first = client.revision(1, &[], base, &[("a", "1")]); + let second = client.revision(1, &[first], base, &[("a", "2")]); + let pass = ci_test(first, "user:ada"); + let fail = Statement { + subject: RecordId::Revision(second), + verdict: Some(Verdict::Fail), + supersedes: Some(pass.encode().id().into()), + ..pass.clone() + }; + client + .store + .write_all(&[fail.encode(), pass.encode()]) + .unwrap(); + + let change = ChangeId::from_bytes([1; 16]); + let pushed = client.sync().push(&[change]).unwrap(); + assert_eq!((pushed.statements, &pushed.unsent[..]), (2, &[][..])); + for statement in [&pass, &fail] { + assert!(server.store.contains(statement.encode().id()).unwrap()); + } + + // One superseding a statement the remote lacks, about a change the + // push leaves out, stays, and the push says so. + let other = client.revision(2, &[], base, &[("b", "1")]); + let old = ci_test(other, "user:ada"); + let newer = Statement { + supersedes: Some(old.encode().id().into()), + ..fail.clone() + }; + client + .store + .write_all(&[old.encode(), newer.encode()]) + .unwrap(); + let pushed = client.sync().push(&[change]).unwrap(); + assert_eq!(pushed.statements, 0); + assert_eq!(pushed.unsent, [StatementId::from(newer.encode().id())]); + } + + #[test] + fn transfers_go_in_parts_in_dependency_order() { + let server = Repo::new(MAIN_RULES); + // Small enough for about one object a part. + let limit = 200; + let client = Client::answered_in_parts(&server, limit); + client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + let base = main_tip(&client.store); + let files: Vec<(String, String)> = (0..20) + .map(|i| (format!("src/{}/{i}", i % 3), format!("content {i}"))) + .collect(); + let files: Vec<_> = files + .iter() + .map(|(p, c)| (p.as_str(), c.as_str())) + .collect(); + let first = client.revision(1, &[], base, &files); + let second = client.revision(1, &[first], base, &files[1..]); + let statement = ci_test(second, "user:ada").encode(); + client + .store + .write_all(std::slice::from_ref(&statement)) + .unwrap(); + // The remote refuses any object sent before one it needs. + let change = ChangeId::from_bytes([1; 16]); + let pushed = client + .sync() + .with_pack_limit(limit) + .push(&[change]) + .unwrap(); + assert_eq!((pushed.revisions, pushed.statements), (2, 1)); + + let other = Client::answered_in_parts(&server, limit); + other.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + let fetched = other.sync().fetch_changes(&[change]).unwrap(); + assert_eq!( + fetched, + Fetched { + revisions: 2, + statements: 1 + } + ); + let snapshot = other.store.get(first).unwrap().snapshot; + let request = FetchSnapshot { + snapshot, + have: Vec::new(), + paths: Vec::new(), + skip: 0, + }; + let part = other.remote.snapshot(&request).unwrap(); + assert!(part.more && part.pack.objects.len() < 10, "{part:?}"); + assert!(other.sync().need(snapshot, &[]).unwrap()); + let empty = build(&other.store, &[]); + assert_eq!(other.store.diff(empty, snapshot).unwrap().len(), 20); + } + + #[test] + fn interrupted_snapshot_fetches_are_fetched_again() { + let server = Repo::new(MAIN_RULES); + let revision = server.change_on_main(1, &[("src/a", "a")]); + let client = Client::new(&server); + client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + client + .sync() + .fetch_changes(&[ChangeId::from_bytes([1; 16])]) + .unwrap(); + let snapshot = client.store.get(revision).unwrap().snapshot; + let empty = build(&client.store, &[]); + + // A fetch cut short once the root was published, not what it holds. + let batch = client.store.batch(); + let fetching = client.store.fetching(snapshot, &batch).unwrap(); + batch + .stage_all(&[server.store.read(snapshot.into()).unwrap()]) + .unwrap(); + batch.commit().unwrap(); + drop(fetching); + assert!(!client.store.holds(snapshot).unwrap()); + assert!(client.sync().need(snapshot, &[]).unwrap(), "fetched again"); + assert!(client.store.holds(snapshot).unwrap()); + assert_eq!(client.store.diff(empty, snapshot).unwrap().len(), 1); + + // One that lacks a tree, unmarked, is repaired whole. + let src = Snapshots::new(&client.store) + .entry_at(snapshot, &RepoPath::new("src").unwrap()) + .unwrap() + .unwrap(); + client.store.remove(&[src.id]).unwrap(); + assert!(client.store.diff(empty, snapshot).is_err()); + client.sync().repair(snapshot).unwrap(); + assert_eq!(client.store.diff(empty, snapshot).unwrap().len(), 1); + } + + #[test] + fn the_remote_events_say_which_changes_moved() { + let server = Repo::new(MAIN_RULES); + let first = server.change_on_main(1, &[("a", "a")]); + let second = server.change_on_main(2, &[("b", "b")]); + let client = Client::new(&server); + let both = [1, 2].map(|change| ChangeId::from_bytes([change; 16])); + client.sync().fetch_changes(&both).unwrap(); + let moved = client.sync().moved(None).unwrap(); + assert_eq!(moved.changes, None, "without a cursor, any change may have"); + let cursor = moved.cursor; + let nothing = client.sync().moved(Some(&cursor)).unwrap(); + assert_eq!(nothing.changes, Some(HashSet::new())); + assert_eq!(nothing.cursor, cursor); + + // A statement about a statement about the second change, and a + // revision of the first. + let annotations = server.annotations(); + let result = annotations + .issue(&ci_test(second, "node:ci"), None) + .unwrap(); + let flaky = Statement { + subject: RecordId::Statement(result), + kind: "flaky".parse().unwrap(), + verdict: None, + inputs: None, + ..ci_test(second, "user:ada") + }; + annotations.issue(&flaky, None).unwrap(); + let tip = main_tip(&server.store); + server.revision(1, &[first], tip, &[("a", "again")]); + let moved = client.sync().moved(Some(&cursor)).unwrap(); + assert_eq!(moved.changes, Some(HashSet::from(both))); + assert!(moved.cursor.next > cursor.next); + } + + #[test] + fn pushes_refuse_stale_changes_and_bases_the_remote_lacks() { + let server = Repo::new(MAIN_RULES); + let shared = server.change_on_main(1, &[("a", "a")]); + let client = Client::new(&server); + client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + let change = ChangeId::from_bytes([1; 16]); + client.sync().fetch_changes(&[change]).unwrap(); + + let base = main_tip(&client.store); + let ours = client.revision(1, &[shared], base, &[("a", "ours")]); + let theirs = server.revision(1, &[shared], base, &[("a", "theirs")]); + let error = client.sync().push(&[change]).unwrap_err(); + assert!( + matches!(error, SyncError::Stale { change: c, head } if c == change && head == theirs), + "{error}" + ); + assert!( + !server.store.contains(ours.into()).unwrap(), + "nothing was sent" + ); + + let local = TargetName::new("local").unwrap(); + let key = NodeKey::from_bytes(&[9; 32]); + let genesis = AcceptanceBody { + target: local.id(), + seq: 0, + prev: None, + state: build(&client.store, &[]), + landed: Vec::new(), + evidence: Vec::new(), + policy: None, + } + .sign(&key); + client.store.append(&local, &genesis).unwrap(); + let on_local = Base::Acceptance(genesis.encode().id().into()); + client.revision(2, &[], on_local, &[("b", "b")]); + let error = client + .sync() + .push(&[ChangeId::from_bytes([2; 16])]) + .unwrap_err(); + assert!( + matches!(&error, SyncError::Unshared { target: Some(target), .. } if *target == local), + "{error}" + ); + } + + #[test] + fn each_part_of_a_push_expects_the_heads_the_parts_before_left() { + let change = ChangeId::from_bytes([1; 16]); + let [r0, r1, r2] = [1, 2, 3].map(|n| RevisionId::from(ObjectId::from_bytes([n; 32]))); + let mut transfer = Transfer { + heads: HashMap::from([(change, vec![r0])]), + revisions: HashMap::from([ + (r1.into(), (change, vec![r0])), + (r2.into(), (change, vec![r1])), + ]), + ..Transfer::default() + }; + let expected = |heads: Vec| vec![Expected { change, heads }]; + assert_eq!(transfer.expect(&[r1.into()]), expected(vec![r0])); + assert_eq!(transfer.expect(&[r2.into()]), expected(vec![r1])); + assert_eq!(transfer.expect(&[]), []); + } + + #[test] + fn revisions_go_with_their_reserved_statements() { + let [first, second] = [1, 2].map(|n| RevisionId::from(ObjectId::from_bytes([n; 32]))); + let revision = |change| Revision { + change: ChangeId::from_bytes([change; 16]), + predecessors: Vec::new(), + base: Base::Revision(first), + deps: Vec::new(), + snapshot: TreeId::from(ObjectId::from_bytes([3; 32])), + description: ObjectId::from_bytes([4; 32]).into(), + author: "user:ada".parse().unwrap(), + }; + let conflict = Statement { + kind: Kind::CONFLICT, + ..ci_test(first, "user:ada") + }; + let result = ci_test(first, "user:ada"); + let abandon = Statement { + kind: Kind::ABANDON, + ..ci_test(second, "user:ada") + }; + // About the second, but it needs the first's conflict: all go in + // one request with both revisions. + let listing = Statement { + supersedes: Some(conflict.encode().id().into()), + ..Statement { + kind: Kind::CONFLICT, + ..ci_test(second, "user:ada") + } + }; + let statements: Vec<_> = [conflict, result, abandon, listing] + .into_iter() + .map(|statement| (statement.encode().id().into(), statement)) + .collect(); + let [conflict, result, abandon, listing] = + [0, 1, 2, 3].map(|at| ObjectId::from(statements[at].0)); + let mut transfer = Transfer::default(); + let others = + transfer.add_revisions(&[(first, revision(1)), (second, revision(2))], statements); + assert_eq!( + others + .iter() + .map(|&(id, _)| id.into()) + .collect::>(), + [result] + ); + assert_eq!( + transfer.ids, + [first.into(), second.into(), conflict, abandon, listing] + ); + assert_eq!( + transfer.joined, + HashSet::from([second.into(), conflict, abandon, listing]) + ); + + // Alone, each goes right after its revision. + let alone = |statement: Statement| (statement.encode().id().into(), statement); + let statements = vec![ + alone(Statement { + kind: Kind::ABANDON, + ..ci_test(first, "user:ada") + }), + alone(Statement { + kind: Kind::DIVERGENT, + ..ci_test(second, "user:ada") + }), + ]; + let [abandon, divergent] = [0, 1].map(|at| ObjectId::from(statements[at].0)); + let mut transfer = Transfer::default(); + transfer.add_revisions(&[(first, revision(1)), (second, revision(2))], statements); + assert_eq!( + transfer.ids, + [first.into(), abandon, second.into(), divergent] + ); + assert_eq!(transfer.joined, HashSet::from([abandon, divergent])); + } + + #[test] + fn statements_about_one_record_go_in_one_request() { + let [first, second] = [1, 2].map(|n| RevisionId::from(ObjectId::from_bytes([n; 32]))); + let pass = ci_test(first, "user:ada"); + let pass_id = StatementId::from(pass.encode().id()); + let fail = Statement { + verdict: Some(Verdict::Fail), + supersedes: Some(pass_id), + ..pass.clone() + }; + let other = ci_test(second, "user:ada"); + // About the result, and what it needs: in one request with it. + let flaky = Statement { + subject: RecordId::Statement(pass_id), + kind: "flaky".parse().unwrap(), + ..other.clone() + }; + let statements: Vec<_> = [pass, other, fail, flaky] + .into_iter() + .map(|statement| (statement.encode().id().into(), statement)) + .collect(); + let mut transfer = Transfer::default(); + transfer.add_statements(&statements); + let [pass, other, fail, flaky] = [0, 1, 2, 3].map(|at| ObjectId::from(statements[at].0)); + assert_eq!(transfer.ids, [pass, fail, flaky, other]); + assert_eq!(transfer.joined, HashSet::from([fail, flaky])); + } + + #[test] + fn a_push_in_parts_never_lands_an_abandoned_revision() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + // Requests of one object, or of the default size, which the + // revision and its content fill. + for fill in [false, true] { + let server = Repo::new(MAIN_RULES); + let client = Client::new(&server); + client.sync().fetch_chains(&[TargetName::MAIN]).unwrap(); + let revision = if fill { + let mut revision = Revision { + change: ChangeId::from_bytes([1; 16]), + predecessors: Vec::new(), + base: main_tip(&client.store), + deps: Vec::new(), + snapshot: build(&client.store, &[("a", "gone")]), + description: EncodedObject::blob(b"").id().into(), + author: "user:ada".parse().unwrap(), + }; + client + .store + .write_all(&[EncodedObject::blob(b"gone")]) + .unwrap(); + let wire = |objects: Vec| Pack { objects }.encode().len(); + let content = client + .store + .added(revision.snapshot, &[], &[]) + .unwrap() + .iter() + .map(|placed| client.store.read(placed.entry.id).unwrap()) + .collect(); + let room = Pack::LIMIT + - wire(content) + - wire(vec![revision.encode(), EncodedObject::blob(b"")]); + let description = EncodedObject::blob(&vec![b'x'; room]); + revision.description = description.id().into(); + let revision = revision.encode(); + client + .store + .write_all(&[description, revision.clone()]) + .unwrap(); + revision.id().into() + } else { + client.revision(1, &[], main_tip(&client.store), &[("a", "gone")]) + }; + let abandonment = Statement { + kind: Kind::ABANDON, + verdict: None, + ..ci_test(revision, "user:ada") + } + .encode(); + client + .store + .write_all(std::slice::from_ref(&abandonment)) + .unwrap(); + // The remote can't take the abandonment, unless this runs as + // root, who writes there anyway: the shard is theirs. + let hex = abandonment.id().to_hex(); + let shard = server.store.root().join("objects/loose").join(&hex[..2]); + std::fs::create_dir_all(&shard).unwrap(); + let root = std::fs::metadata(&shard).unwrap().uid() == 0; + std::fs::set_permissions(&shard, std::fs::Permissions::from_mode(0o555)).unwrap(); + let limit = if fill { Pack::LIMIT } else { 1 }; + let pushed = client + .sync() + .with_pack_limit(limit) + .push(&[ChangeId::from_bytes([1; 16])]); + std::fs::set_permissions(&shard, std::fs::Permissions::from_mode(0o755)).unwrap(); + if !root { + assert!(pushed.is_err(), "{pushed:?}"); + assert!(!server.store.contains(revision.into()).unwrap()); + } + // Pushed again, it's abandoned there too. + let change = ChangeId::from_bytes([1; 16]); + client + .sync() + .with_pack_limit(limit) + .push(&[change]) + .unwrap(); + let landed = + server + .queue() + .land_change(&mut server.annotations(), change, &TargetName::MAIN); + assert!( + matches!(landed, Err(crate::queue::LandError::Abandoned(_))), + "{landed:?}" + ); + } + } + + #[test] + fn queue_rebases_and_pushes_never_make_a_change_diverge() { + let server = Repo::new(MAIN_RULES); + server.change_on_main(1, &[("a", "a")]); + let original = server.change_on_main(2, &[("b", "b")]); + server.land(1); + let change = ChangeId::from_bytes([2; 16]); + // A push whose check of the heads passed, about to publish a + // successor of the head, while the queue rebases that head. + let pushing = server.store.lock_pushes().unwrap(); + let description = EncodedObject::blob(b"concurrent edit"); + let successor = Revision { + predecessors: vec![original], + description: description.id().into(), + ..server.store.get(original).unwrap() + } + .encode(); + let landed = std::thread::scope(|scope| { + let landing = scope.spawn(|| { + server + .queue() + .land_change(&mut server.annotations(), change, &TargetName::MAIN) + }); + std::thread::sleep(std::time::Duration::from_millis(100)); + server + .store + .write_all(&[description, successor.clone()]) + .unwrap(); + drop(pushing); + landing.join().unwrap() + }); + // The queue waited for the push, and found the change moved: or + // had it come late, rebased the successor. + let heads = Index::open(&server.store).unwrap().heads(change); + assert_eq!(heads.len(), 1, "{landed:?}"); + assert!( + matches!(landed, Err(crate::queue::LandError::Moved(moved)) if moved == change) + || landed.is_ok(), + "{landed:?}" + ); + } + + #[test] + fn cursors_of_another_op_log_read_everything() { + let server = Repo::new(MAIN_RULES); + let first = server.change_on_main(1, &[("a", "a")]); + let change = ChangeId::from_bytes([1; 16]); + let client = Client::new(&server); + let fetch = client + .sync() + .fetch(&[TargetName::MAIN], &[change], None) + .unwrap(); + // The remote was restored, with a new op log, and has a revision + // since: a cursor of the old log, even ahead of the new one's + // end, finds it. + let cursor = Cursor { + generation: "restored".to_owned(), + next: fetch.cursor.next + 10_000, + }; + let newer = server.revision(1, &[first], main_tip(&server.store), &[("a", "new")]); + let fetch = client.sync().fetch(&[], &[], Some(&cursor)).unwrap(); + assert!(client.store.contains(newer.into()).unwrap()); + assert_ne!(fetch.cursor.generation, cursor.generation); + } +} diff --git a/apps/origin-src/src/targets.rs b/apps/origin-src/src/targets.rs index 49431a1d..92d0629c 100644 --- a/apps/origin-src/src/targets.rs +++ b/apps/origin-src/src/targets.rs @@ -25,8 +25,8 @@ use std::slice; use origin_platform::{DurableWrite, LockMode, LockedFile, Platform}; use origin_shared::{ - Acceptance, AcceptanceId, Base, Head, Record, Revision, RevisionId, Source, TargetId, - TargetName, + Acceptance, AcceptanceId, Base, EncodedObject, Head, Record, Revision, RevisionId, Source, + TargetId, TargetName, }; use crate::oplog::Effect; @@ -104,6 +104,20 @@ impl TargetFiles { .collect() } + /// The complete entries of a locked log from position `since` on. + fn entries_since(&self, log: &File, since: u64) -> Result, StoreError> { + let mut bytes = Vec::new(); + let mut reader = log; + reader + .seek(SeekFrom::Start(since * Self::LINE as u64)) + .and_then(|_| reader.read_to_end(&mut bytes)) + .map_err(|source| StoreError::io(&self.log(), source))?; + Ok((since..) + .zip(self.parse_log(&bytes)?) + .map(|(seq, acceptance)| Head { seq, acceptance }) + .collect()) + } + /// The last complete entry of a locked log, after cutting off a partial /// line left by a crash. fn last_entry(&self, log: &File) -> Result, StoreError> { @@ -244,6 +258,26 @@ impl Store

{ .then_some(next)) } + /// The target the stack of `revision` is on: that of the acceptance at + /// the bottom of its stack, found by following bases. + /// + /// # Errors + /// + /// Fails if a record of the stack cannot be read, or the store doesn't + /// know the target's name. + pub fn stack_target(&self, revision: RevisionId) -> Result { + let mut base = self.get(revision)?.base; + let acceptance = loop { + match base { + Base::Revision(id) => base = self.get(id)?.base, + Base::Acceptance(id) => break id, + } + }; + let target = self.get(acceptance)?.body().target; + self.target_name(target)? + .ok_or(StoreError::NotFound(acceptance.into())) + } + /// The name of the target `id`, or `None` if it doesn't exist. /// /// # Errors @@ -290,40 +324,74 @@ impl Store

{ } /// Appends `acceptance` to the chain of `target`, creating the target - /// with its first acceptance. - /// - /// This is a compare-and-swap: the acceptance must follow the target's - /// latest one, with the next sequence number and naming it as `prev`. - /// Concurrent appends are serialized, and all but one of those following - /// the same acceptance fail. Verifying the signature is the caller's job, - /// since only it knows the target's authority. + /// with its first acceptance. See [`Store::append_all`]. /// /// # Errors /// - /// Fails with [`StoreError::Conflict`] if the acceptance doesn't follow - /// the latest one, [`StoreError::WrongTarget`] if it belongs to another - /// target, or if the target's files cannot be locked, read or written. + /// Fails like [`Store::append_all`]. pub fn append( &self, target: &TargetName, acceptance: &Acceptance, ) -> Result { - let body = acceptance.body(); - if body.target != target.id() { - return Err(StoreError::WrongTarget(target.clone())); - } + self.append_all(target, slice::from_ref(acceptance), &[])?; + Ok(acceptance.encode().id().into()) + } + + /// Appends `acceptances`, in order, to the chain of `target`, creating + /// the target with its first acceptance. + /// `with`, the records the acceptances rely on, such as promoted + /// statements, are written in the same batch, so that costs no extra + /// sync. + /// + /// This is a compare-and-swap: the first acceptance must follow the + /// target's latest one, with the next sequence number and naming it as + /// `prev`, and each the one before it. Concurrent appends are + /// serialized, and all but one of those following the same acceptance + /// fail. Verifying the signatures is the caller's job, since only it + /// knows the target's authority. + /// + /// The op log says the target advanced to each acceptance appended, + /// even one the store already had, such as one fetched before as + /// another target's landing: its `target.advanced` event comes from + /// that entry. + /// + /// # Errors + /// + /// Fails with [`StoreError::Conflict`] if an acceptance doesn't follow + /// the one before it, [`StoreError::WrongTarget`] if one belongs to + /// another target, or if the target's files cannot be locked, read or + /// written. + pub fn append_all( + &self, + target: &TargetName, + acceptances: &[Acceptance], + with: &[EncodedObject], + ) -> Result<(), StoreError> { + let Some(first) = acceptances.first() else { + return Ok(()); + }; // A new policy changes which payloads collection keeps, so it never // lands while collection, which reads the policy first, runs. let _held = (*target == TargetName::POLICY) .then(|| self.hold_objects()) .transpose()?; - let encoded = acceptance.encode(); - // The acceptance is durable before the log refers to it. - self.write_all(slice::from_ref(&encoded))?; - let id = AcceptanceId::from(encoded.id()); + let mut records = with.to_vec(); + let mut ids = Vec::with_capacity(acceptances.len()); + for acceptance in acceptances { + if acceptance.body().target != target.id() { + return Err(StoreError::WrongTarget(target.clone())); + } + let encoded = acceptance.encode(); + ids.push(AcceptanceId::from(encoded.id())); + records.push(encoded); + } + // The acceptances are durable before the log refers to them. + self.write_all(&records)?; - let files = self.target_files(body.target); - fs::create_dir_all(&files.dir).map_err(|source| StoreError::io(&files.dir, source))?; + let files = self.target_files(target.id()); + // Before its first head, so a target that exists is never nameless. + self.name_target(target)?; let log = LockedFile::open(&self.platform, &files.log(), LockMode::Exclusive)?; let latest = files.last_entry(&log)?; let head = files.read_head()?; @@ -334,64 +402,105 @@ impl Store

{ // durable acceptances, but may not be durable themselves. Some(latest) if head.is_none_or(|head| head.seq < latest.seq) => { self.platform.sync_file(&log).map_err(log_error)?; - self.publish(&files, target, latest)?; + let next = head.map_or(0, |head| head.seq + 1); + let advanced = files.entries_since(&log, next)?; + self.publish(&files, target, &advanced)?; } _ => return Err(StoreError::Malformed(files.log())), } } - let follows = match latest { - None => body.seq == 0 && body.prev.is_none(), - Some(latest) => body.seq == latest.seq + 1 && body.prev == Some(latest.acceptance), - }; - if !follows { - return Err(StoreError::Conflict { - target: target.clone(), - expected: body.seq, - actual: latest, + let mut previous = latest; + for (acceptance, &id) in acceptances.iter().zip(&ids) { + let body = acceptance.body(); + let follows = match previous { + None => body.seq == 0 && body.prev.is_none(), + Some(previous) => { + body.seq == previous.seq + 1 && body.prev == Some(previous.acceptance) + } + }; + if !follows { + return Err(StoreError::Conflict { + target: target.clone(), + expected: body.seq, + actual: latest, + }); + } + previous = Some(Head { + seq: body.seq, + acceptance: id, }); } + let mut lines = String::with_capacity(ids.len() * TargetFiles::LINE); + for id in &ids { + lines.push_str(&id.to_hex()); + lines.push('\n'); + } let mut writer = &*log; writer - .seek(SeekFrom::Start(body.seq * TargetFiles::LINE as u64)) - .and_then(|_| writeln!(writer, "{id}")) + .seek(SeekFrom::Start(first.body().seq * TargetFiles::LINE as u64)) + .and_then(|_| writer.write_all(lines.as_bytes())) .map_err(log_error)?; self.platform.sync_file(&log).map_err(log_error)?; - let head = Head { - seq: body.seq, - acceptance: id, - }; - self.publish(&files, target, head)?; - Ok(id) + let advanced: Vec = acceptances + .iter() + .zip(&ids) + .map(|(acceptance, &id)| Head { + seq: acceptance.body().seq, + acceptance: id, + }) + .collect(); + self.publish(&files, target, &advanced) + } + + /// Remembers the name of `target`, before its first acceptance is + /// appended, or for one this store doesn't follow, so that acceptances + /// of it it holds, such as another target's landings, can be described. + /// The target doesn't exist for this store until its first acceptance + /// is appended. + /// + /// # Errors + /// + /// Fails if the name cannot be written. + pub fn name_target(&self, target: &TargetName) -> Result<(), StoreError> { + let files = self.target_files(target.id()); + if files.name().exists() { + return Ok(()); + } + fs::create_dir_all(&files.dir).map_err(|source| StoreError::io(&files.dir, source))?; + let write = self.durable_write(); + write.stage(files.name(), target.as_str().as_bytes())?; + write.commit()?; + self.sync_dir(&self.targets_dir()) } - /// Durably replaces the head of a target, logging that it advanced, from - /// which its event comes. Publishing the first - /// acceptance creates the target: its name is written too, and its - /// directory made durable. + /// Durably replaces the head of a target with the last of `advanced`, + /// the acceptances appended, logging that it advanced to each, which + /// their events come from. Publishing its first head, which may name a + /// later acceptance than the first, creates the target, which + /// [`Store::name_target`] named before. fn publish( &self, files: &TargetFiles, target: &TargetName, - head: Head, + advanced: &[Head], ) -> Result<(), StoreError> { + let Some(&head) = advanced.last() else { + return Ok(()); + }; let write = self.durable_write(); let line = format!("{} {}\n", head.seq, head.acceptance); write.stage(files.head(), line.as_bytes())?; - let created = head.seq == 0; - if created { - write.stage(files.name(), target.as_str().as_bytes())?; - } - let advanced = Effect::Advanced { - target: target.clone(), - seq: head.seq, - acceptance: head.acceptance, - }; - self.commit_logged(write, &[advanced])?; - if created { - self.sync_dir(&self.targets_dir())?; - } + let advanced: Vec<_> = advanced + .iter() + .map(|advanced| Effect::Advanced { + target: target.clone(), + seq: advanced.seq, + acceptance: advanced.acceptance, + }) + .collect(); + self.commit_logged(write, &advanced)?; Ok(()) } } @@ -442,6 +551,23 @@ mod tests { .collect() } + #[test] + fn appending_several_acceptances_creates_the_target() { + let (_dir, store) = store(); + let main = TargetName::MAIN; + let first = acceptance(&main, None); + let second = acceptance( + &main, + Some(Head { + seq: 0, + acceptance: first.encode().id().into(), + }), + ); + store.append_all(&main, &[first, second], &[]).unwrap(); + assert_eq!(store.targets().unwrap(), [TargetName::MAIN]); + assert_eq!(store.head(&main).unwrap().unwrap().seq, 1); + } + #[test] fn appends_form_a_chain() { let (_dir, store) = store(); @@ -587,6 +713,50 @@ mod tests { assert_eq!(store.log(&main).unwrap().len(), 4); } + #[test] + fn repairs_advance_the_target_to_every_acceptance_left_unpublished() { + let (_dir, store) = store(); + let main = TargetName::MAIN; + let ids = land(&store, &main, 1); + let before = store.oplog().last_seq().unwrap(); + // A crash after appending three acceptances to the log at once, + // before publishing the head. + let files = store.target_files(main.id()); + let mut log = fs::OpenOptions::new() + .append(true) + .open(files.log()) + .unwrap(); + let mut head = Head { + seq: 0, + acceptance: ids[0], + }; + for _ in 0..3 { + let next = acceptance(&main, Some(head)).encode(); + store.write_all(slice::from_ref(&next)).unwrap(); + head = Head { + seq: head.seq + 1, + acceptance: next.id().into(), + }; + writeln!(log, "{}", head.acceptance).unwrap(); + } + + let stale = acceptance(&main, store.head(&main).unwrap()); + assert!(store.append(&main, &stale).is_err(), "repaired first"); + let advanced: Vec<_> = store + .oplog() + .after(before) + .unwrap() + .entries + .into_iter() + .filter_map(|entry| match entry.effect { + Effect::Advanced { seq, .. } => Some(seq), + _ => None, + }) + .collect(); + assert_eq!(advanced, [1, 2, 3], "one event per acceptance"); + assert_eq!(store.head(&main).unwrap(), Some(head)); + } + #[test] fn repairs_sync_the_log_before_publishing_it() { let dir = tempfile::tempdir().unwrap(); @@ -627,6 +797,25 @@ mod tests { ); } + #[test] + fn targets_are_named_before_their_first_head() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Recorder::default()); + store.init().unwrap(); + let main = TargetName::MAIN; + store.platform.0.lock().unwrap().clear(); + + store.append(&main, &acceptance(&main, None)).unwrap(); + let targets = Synced::Dir(store.targets_dir()); + let synced = store.platform.0.into_inner().unwrap(); + let named = synced.iter().position(|synced| *synced == targets); + // The head, and the op log entry saying it advanced. + let published = synced + .iter() + .rposition(|synced| *synced == Synced::Batch(2)); + assert!(named.unwrap() < published.unwrap(), "{synced:?}"); + } + #[test] fn acceptances_are_durable_before_the_log_refers_to_them() { let dir = tempfile::tempdir().unwrap(); diff --git a/apps/origin-src/src/tree.rs b/apps/origin-src/src/tree.rs index 19e144d1..7ea9d1f4 100644 --- a/apps/origin-src/src/tree.rs +++ b/apps/origin-src/src/tree.rs @@ -2,6 +2,9 @@ use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; +use std::fs::{self, File}; +use std::io; +use std::path::PathBuf; use origin_platform::Platform; use origin_shared::{ @@ -10,7 +13,7 @@ use origin_shared::{ }; use crate::conflict::{Conflicted, Conflicts}; -use crate::store::{Store, StoreError}; +use crate::store::{Batch, Store, StoreError}; /// A file that differs between two trees. #[derive(Clone, Debug, PartialEq, Eq)] @@ -433,6 +436,193 @@ impl Store

{ } } +/// A snapshot being fetched, marked durably in the store's `fetching/`, so +/// an interrupted fetch, which published trees before what they hold, is +/// never taken for a whole snapshot: see [`Store::holds`]. Dropping it +/// without [`Fetching::done`] leaves the mark, and fetching the snapshot +/// again completes it. +#[derive(Debug)] +pub struct Fetching { + path: PathBuf, +} + +impl Fetching { + /// Removes the mark, once everything fetched is durable. A crash that + /// loses the removal only fetches the snapshot again. + /// + /// # Errors + /// + /// Fails if the mark cannot be removed. + pub fn done(self) -> Result<(), StoreError> { + match fs::remove_file(&self.path) { + Err(error) if error.kind() != io::ErrorKind::NotFound => { + Err(StoreError::io(&self.path, error)) + } + _ => Ok(()), + } + } +} + +impl Store

{ + /// The directory marking the snapshots being fetched. + const FETCHING: &str = "fetching"; + + fn fetching_path(&self, snapshot: TreeId) -> PathBuf { + self.root().join(Self::FETCHING).join(snapshot.to_hex()) + } + + /// Whether the store holds `snapshot` whole: its root, and no fetch of + /// it that didn't finish (see [`Fetching`]). + /// + /// # Errors + /// + /// Fails like [`Store::contains`]. + pub fn holds(&self, snapshot: TreeId) -> Result { + Ok(self.contains(snapshot.into())? && !self.fetching_path(snapshot).exists()) + } + + /// Those of `snapshots` the store [holds](Store::holds) whole, in + /// order. + /// + /// # Errors + /// + /// Fails like [`Store::contains`]. + pub fn held_whole( + &self, + snapshots: impl IntoIterator, + ) -> Result, StoreError> { + let mut held = Vec::new(); + for snapshot in snapshots { + if self.holds(snapshot)? { + held.push(snapshot); + } + } + Ok(held) + } + + /// Marks `snapshot` as being fetched, durably before anything `batch` + /// stages is published, at no sync of its own: the batch syncs the mark + /// with what it stages. + /// + /// # Errors + /// + /// Fails if the mark, or the directory holding it, cannot be created. + pub fn fetching(&self, snapshot: TreeId, batch: &Batch<'_, P>) -> Result { + let dir = self.root().join(Self::FETCHING); + if !dir.is_dir() { + fs::create_dir_all(&dir).map_err(|source| StoreError::io(&dir, source))?; + self.sync_dir(self.root())?; + } + let path = dir.join(snapshot.to_hex()); + File::create(&path).map_err(|source| StoreError::io(&path, source))?; + batch.sync_first(&path); + batch.sync_first(dir); + Ok(Fetching { path }) + } +} + +/// An object of a snapshot, and where it is: [`Store::added`] lists them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Placed { + /// Its path: the root for the snapshot's own tree. + pub path: RepoPath, + /// The object, and its kind. + pub entry: TreeEntry, +} + +/// Which part of a snapshot [`Store::added`] walks. +struct Scope<'p>(&'p [RepoPath]); + +impl Scope<'_> { + /// Whether everything at `path` and under it is in scope. + fn covers(&self, path: &RepoPath) -> bool { + self.0.is_empty() || self.0.iter().any(|prefix| path.starts_with(prefix)) + } + + /// Whether `path` is a directory on the way to something in scope. + fn leads_in(&self, path: &RepoPath) -> bool { + self.0.iter().any(|prefix| prefix.starts_with(path)) + } +} + +impl Store

{ + /// The objects of `snapshot` that none of the snapshots `have` holds at + /// the same path, parents before children: what someone holding `have` + /// lacks of it. With `paths`, only those under them, and the trees on + /// the way. Subtrees a `have` holds whole are skipped without being + /// read, so the cost follows the size of the change. Objects appear + /// once, at their first path. + /// + /// # Errors + /// + /// Fails if a tree of `snapshot`, or of a `have`, cannot be read. + pub fn added( + &self, + snapshot: TreeId, + have: &[TreeId], + paths: &[RepoPath], + ) -> Result, StoreError> { + let mut added = Vec::new(); + let mut seen = HashSet::new(); + self.add_dir( + &RepoPath::root(), + snapshot, + have, + &Scope(paths), + &mut seen, + &mut added, + )?; + Ok(added) + } + + fn add_dir( + &self, + dir: &RepoPath, + tree: TreeId, + have: &[TreeId], + scope: &Scope<'_>, + seen: &mut HashSet, + added: &mut Vec, + ) -> Result<(), StoreError> { + if have.contains(&tree) { + return Ok(()); + } + if seen.insert(tree.into()) { + added.push(Placed { + path: dir.clone(), + entry: TreeEntry { + kind: EntryKind::Tree, + id: tree.into(), + }, + }); + } + let have: Vec = have + .iter() + .map(|&id| self.get(id)) + .collect::>()?; + for (name, &entry) in &self.get(tree)? { + let path = dir.join(name); + let covered = scope.covers(&path); + if !(covered || entry.kind.is_tree() && scope.leads_in(&path)) { + continue; + } + let held = have.iter().filter_map(|tree| tree.get(name.as_str())); + if held.clone().any(|held| *held == entry) { + continue; + } + match entry.subtree() { + Some(subtree) => { + let held: Vec<_> = held.filter_map(|held| held.subtree()).collect(); + self.add_dir(&path, subtree, &held, scope, seen, added)?; + } + None if seen.insert(entry.id) => added.push(Placed { path, entry }), + None => {} + } + } + Ok(()) + } +} + #[cfg(test)] pub(crate) mod tests { use std::collections::BTreeMap; @@ -560,6 +750,67 @@ pub(crate) mod tests { assert_eq!(store.diff(empty, old).unwrap().len(), 5); } + #[test] + fn lists_what_the_haves_lack() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::new(dir.path().join(".origin"), Native::default()); + store.init().unwrap(); + let old = build( + &store, + &[ + ("same/a", "1"), + ("edit/b", "2"), + ("file", "3"), + ("docs/x", "4"), + ], + ); + let new = build( + &store, + &[ + ("same/a", "1"), + ("edit/b", "changed"), + ("file/now-dir", "4"), + ("docs/x", "4"), + ("docs/copy", "4"), + ], + ); + let paths = |placed: Vec| -> Vec { + placed + .iter() + .map(|placed| placed.path.to_string()) + .collect() + }; + + assert_eq!( + paths(store.added(new, &[old], &[]).unwrap()), + ["/", "docs", "docs/copy", "edit", "edit/b", "file"], + "`file/now-dir` has the content of `docs/copy`, listed first" + ); + assert_eq!( + store.added(new, &[], &[]).unwrap().len(), + 8, + "5 trees, and 5 files of 3 contents" + ); + assert_eq!(store.added(new, &[new], &[]).unwrap(), []); + let empty = build(&store, &[]); + assert_eq!( + paths(store.added(new, &[empty, old], &[]).unwrap()), + paths(store.added(new, &[old], &[]).unwrap()), + "any have holding an object is enough" + ); + + let only = |path: &str| [RepoPath::new(path).unwrap()]; + assert_eq!( + paths(store.added(new, &[], &only("edit/b")).unwrap()), + ["/", "edit", "edit/b"], + "the trees on the way, then the path" + ); + assert_eq!( + paths(store.added(new, &[old], &only("same")).unwrap()), + ["/"] + ); + } + /// The files of `tree`, by path, with their contents' ids. fn files(store: &Store, tree: TreeId) -> Vec<(String, ObjectId)> { let empty = build(store, &[]); diff --git a/apps/origin-src/src/workspace/land.rs b/apps/origin-src/src/workspace/land.rs index 0dc23738..21a30cfd 100644 --- a/apps/origin-src/src/workspace/land.rs +++ b/apps/origin-src/src/workspace/land.rs @@ -3,7 +3,10 @@ //! //! They go through the repository's //! [`OriginService`](crate::service::OriginService), whose queue runs in -//! this process, as this node. After a landing, the +//! this process, as this node, unless the repository has a remote that has +//! the target: then they go to the remote, whose node must be the target's +//! authority, and `land` pushes the stack first, and fetches the target's +//! chain and the stack's records after. After a landing, the //! working copy of the workspace that ran it follows its change: when it //! landed, or the target already had it, the working copy starts a new //! change on the target's tip, so later edits are made on what the target @@ -18,7 +21,7 @@ //! the working copy doesn't land until they change or are accepted with //! `origin resolve` (see [`conflict`](crate::conflict)). -use std::mem; +use std::{mem, slice}; use origin_shared::api::{LandRequest, Landings, NewTarget}; use origin_shared::landing::{Blocked, Landing, Pending, Proposal}; @@ -30,6 +33,7 @@ use super::{Behind, WorkingCopy, Workspace, WorkspaceError}; use crate::conflict::Conflicts; use crate::index::Index; use crate::oplog::Why; +use crate::remote::Remote; use crate::service::Caller; use crate::tree::Merge; @@ -49,13 +53,36 @@ impl Workspace { ) -> Result<(), WorkspaceError> { let mut working_copy = self.save_draft()?; let change = working_copy.find_change(change.unwrap_or("@"))?; + let onto = if let Some(onto) = onto { + onto + } else { + let head = Self::single_head(&working_copy.index, change)?; + self.store.stack_target(head)? + }; let request = LandRequest { proposal: Proposal::Change(change), - onto, + onto: Some(onto.clone()), + }; + let Landings { onto, landings } = match self.remote_queue(&onto)? { + Some(remote) => { + let sync = self.sync(remote); + sync.push(&[change])?; + let landed = remote.land(&request)?; + sync.fetch_chains(slice::from_ref(&landed.onto))?; + let mut changes = vec![change]; + changes.extend(landed.landings.iter().filter_map( + |landing| match landing.proposal() { + Proposal::Change(change) => Some(*change), + Proposal::Target(_) => None, + }, + )); + sync.fetch_changes(&changes)?; + landed + } + None => self + .service(Some(mem::take(&mut working_copy.index)))? + .land(&Caller::Owner, &request)?, }; - let Landings { onto, landings } = self - .service(Some(mem::take(&mut working_copy.index)))? - .land(&Caller::Owner, &request)?; for landing in &landings { self.print_landing(landing, &onto)?; } @@ -86,20 +113,31 @@ impl Workspace { proposal: Proposal::Target(from.clone()), onto: Some(onto.clone()), }; - let landed = self.service(None)?.land(&Caller::Owner, &request)?; + let landed = match self.remote_queue(onto)? { + Some(remote) => { + let landed = remote.land(&request)?; + self.sync(remote).fetch_chains(slice::from_ref(onto))?; + landed + } + None => self.service(None)?.land(&Caller::Owner, &request)?, + }; for landing in &landed.landings { self.print_landing(landing, onto)?; } Ok(()) } - /// `origin land --status`: lists the landings that wait, and why. + /// `origin land --status`: lists the landings that wait, and why: in + /// this node's queues, then in the remote's. /// /// # Errors /// /// Fails if the queues cannot be read. pub fn land_status(&self) -> Result<(), WorkspaceError> { - let pending = self.service(None)?.landings(&Caller::Owner)?; + let mut pending = self.service(None)?.landings(&Caller::Owner)?; + if let Some(remote) = self.remote()? { + pending.extend(remote.landings()?); + } if pending.is_empty() { println!("No landing waits."); } @@ -121,9 +159,16 @@ impl Workspace { name: name.clone(), from: from.clone(), }; - let head = self - .service(None)? - .create_target(&Caller::Owner, &request)?; + let head = match self.remote_queue(name)? { + Some(remote) => { + let head = remote.create_target(&request)?; + self.sync(remote).fetch_chains(slice::from_ref(name))?; + head + } + None => self + .service(None)? + .create_target(&Caller::Owner, &request)?, + }; let landed = self.store.get(head.acceptance)?.body().landed.clone(); let from = match landed[..] { [Source::Target(from)] => self.describe_acceptance(from)?, @@ -134,13 +179,22 @@ impl Workspace { } /// `origin targets`: lists every target with its tip, its authority and - /// what landing on it requires. + /// what landing on it requires: the remote's, if the repository has + /// one, since this one only holds those it follows, and this one's own. /// /// # Errors /// - /// Fails if the targets or the policy cannot be read. + /// Fails if the remote fails, or the targets or the policy cannot be + /// read. pub fn targets(&self) -> Result<(), WorkspaceError> { - for target in self.service(None)?.targets(&Caller::Owner)? { + let mut targets = self.service(None)?.targets(&Caller::Owner)?; + if let Some(remote) = self.remote()? { + let remote = remote.targets()?; + targets.retain(|own| !remote.iter().any(|target| target.name == own.name)); + targets.extend(remote); + targets.sort_by(|a, b| a.name.cmp(&b.name)); + } + for target in targets { let authority = target .authority .map_or_else(|| "no authority".to_owned(), |node| format!("node:{node}")); @@ -157,6 +211,20 @@ impl Workspace { Ok(()) } + /// The remote to submit landings on `target` to, or `None` if they land + /// here: if the repository has no remote, or the target is this + /// repository's own, which the remote doesn't have. The chains of the + /// others are copies of the remote's, which only the remote extends, + /// even when this node is their authority. + fn remote_queue(&self, target: &TargetName) -> Result, WorkspaceError> { + let Some(remote) = self.remote()? else { + return Ok(None); + }; + let own = self.store.head(target)?.is_some() + && !remote.targets()?.iter().any(|known| known.name == *target); + Ok((!own).then_some(remote)) + } + /// `origin new [description] --on `: starts a change on the tip /// of a target. /// @@ -171,17 +239,22 @@ impl Workspace { self.start_on(&mut working_copy, target, description, files) } - /// `origin pull`: moves the working copy where its change went without - /// it, merging the edits made since: onto the tip of the target it + /// `origin pull`: fetches, if the repository has a remote, then moves + /// the working copy where its change went without it, merging the edits + /// made since: onto the tip of the target it /// landed on, as a new change, or onto the landing queue's rebase of it. /// Otherwise rebases it onto the tip of the target its stack is on: /// once the change below it landed, or once the target it's on moved. /// /// # Errors /// - /// Fails if the working copy cannot be snapshotted, a file was edited - /// while it moved, or the records cannot be read or written. + /// Fails like [`Workspace::fetch`], or if the working copy cannot be + /// snapshotted, a file was edited while it moved, or the records cannot + /// be read or written. pub fn pull(&mut self) -> Result<(), WorkspaceError> { + if self.remote()?.is_some() { + self.fetch(&[], &[])?; + } let mut working_copy = self.save_draft()?; if self.follow(&mut working_copy)? { return Ok(()); @@ -218,6 +291,7 @@ impl Workspace { .head(target)? .ok_or_else(|| WorkspaceError::UnknownTarget(target.clone()))?; let state = self.store.get(tip.acceptance)?.body().state; + self.need(&[state])?; let tip_label = format!("{target} #{}", tip.seq); let merge = self.store.merge_marked( Some(base), @@ -318,7 +392,7 @@ impl Workspace { /// Rebases the working copy, made on `base`, onto the tip of `onto`, as /// the module explains. - fn rebase_onto_tip( + pub(super) fn rebase_onto_tip( &mut self, working_copy: &mut WorkingCopy, base: TreeId, @@ -329,6 +403,7 @@ impl Workspace { .head(onto)? .ok_or_else(|| WorkspaceError::UnknownTarget(onto.clone()))?; let state = self.store.get(tip.acceptance)?.body().state; + self.need(&[state])?; let current = &working_copy.revision; let tip_label = format!("{onto} #{}", tip.seq); let merge = self.store.merge_marked( diff --git a/apps/origin-src/src/workspace/mod.rs b/apps/origin-src/src/workspace/mod.rs index 80620a16..36bad9db 100644 --- a/apps/origin-src/src/workspace/mod.rs +++ b/apps/origin-src/src/workspace/mod.rs @@ -67,6 +67,7 @@ mod annotate; mod land; mod ops; mod registry; +mod remote; pub use annotate::{Annotated, Payload}; @@ -75,10 +76,10 @@ use std::fs; use std::io; use std::mem; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use origin_platform::{LockedFile, Native}; -use origin_shared::api::{RevisionEntry, Revisions}; +use origin_shared::api::{LogQuery, RevisionEntry, Revisions}; use origin_shared::{ AcceptanceId, Base, BlobId, ChangeId, EncodedObject, Kind, Principal, Record, RecordId, RepoPath, Revision, RevisionId, Source, Statement, StatementId, TargetId, TargetName, Tree, @@ -94,8 +95,11 @@ use crate::oplog::{Actor, Effect, Move, Why}; use crate::output::FileDiff; use crate::policy::{Policy, PolicyError}; use crate::queue::{LandError, Queue}; +use crate::remote::{Remote, RemoteError}; use crate::service::{Caller, Node, OriginService, ServiceError}; use crate::store::{Store, StoreError}; +use crate::sync::SyncError; +use crate::tree::FileChange; use crate::worktree::{CloneSource, SnapshotError, StatCache, Worktree}; /// A working copy and its repository. @@ -112,6 +116,8 @@ pub struct Workspace { /// The `moving` of a move a crash cut short, which couldn't be read: /// the working copy doesn't move while it's there. unreadable_move: Option, + /// The repository's remote, once opened: see [`Workspace::remote`]. + remote: OnceLock>, } /// The working copy as of a snapshot. @@ -265,14 +271,7 @@ impl Workspace { author: config.user(), }; let encoded = revision.encode(); - let mut workspace = Self { - root, - store, - config, - revision: encoded.id().into(), - unlogged: None, - unreadable_move: None, - }; + let mut workspace = Self::at(root, store, config, encoded.id().into()); workspace.write(&[vec![empty_description], vec![encoded]])?; workspace.publish_pointer(None, workspace.revision, Why::Command)?; println!( @@ -319,17 +318,25 @@ impl Workspace { }; let revision = Self::read_pointer(root)?.ok_or_else(|| WorkspaceError::Incomplete(root.to_owned()))?; - let mut workspace = Self { - root: root.to_owned(), - store: Store::open_writable(store, Native::default())? - .with_actor(actor.in_workspace(root)), + let store = Store::open_writable(store, Native::default())?; + let store = store.with_actor(actor.in_workspace(root)); + let mut workspace = Self::at(root.to_owned(), store, config, revision); + workspace.finish_move()?; + Ok(workspace) + } + + /// The working copy at `root`, of the repository in `store`, which is + /// `revision`. + fn at(root: PathBuf, store: Store, config: UserConfig, revision: RevisionId) -> Self { + Self { + root, + store, config, revision, unlogged: None, unreadable_move: None, - }; - workspace.finish_move()?; - Ok(workspace) + remote: OnceLock::new(), + } } /// Finishes the move `moving` records, if a crash cut it short, as the @@ -423,17 +430,26 @@ impl Workspace { /// that the API also offers go through. It finds records through /// `index`, if the command just read it, or else reads it once needed. fn service(&self, index: Option) -> Result { - let node = Node { - name: self.config.node().clone(), - key: Arc::new(self.config.node_key()?), - }; - let service = OriginService::new(self.store.clone(), node); + let service = OriginService::new(self.store.clone(), self.node()?); Ok(match index { Some(index) => service.with_index(index), None => service, }) } + /// This node, with its key. + fn node(&self) -> Result { + Self::node_of(&self.config) + } + + /// The node `config` names, with its key. + fn node_of(config: &UserConfig) -> Result { + Ok(Node { + name: config.node().clone(), + key: Arc::new(config.node_key()?), + }) + } + /// Snapshots the working copy into a draft, recording nothing, for the /// commands that only read it. fn snapshot(&mut self) -> Result { @@ -744,8 +760,14 @@ impl Workspace { ) -> Result<(), WorkspaceError> { // Before changing any file. self.may_move()?; + // Fetched first, so a fetch that fails moves nothing. + let changes = if revision.snapshot == working_copy.files { + Vec::new() + } else { + self.changes(working_copy.files, revision.snapshot)? + }; // The files stay as they are: there's nothing to finish. - if revision.snapshot == working_copy.files { + if changes.is_empty() { self.write(unwritten)?; return self.point_at(to, why); } @@ -756,11 +778,7 @@ impl Workspace { why, }; self.publish_moving(&moving)?; - let checkout = self.checkout( - working_copy.files, - revision.snapshot, - Some(&working_copy.cache), - ); + let checkout = self.checkout(&changes, Some(&working_copy.cache)); if let Err(error) = checkout { // So a crash finishes taking the files back, not the move, whose // revision may exist. Best effort: the checkout's error says @@ -807,17 +825,34 @@ impl Workspace { Ok(Worktree::new(&self.root, &self.store).restore(&changes)?) } - /// Makes the files match `to`, given that they match `from`, durably, - /// cloning those the other workspaces have. With `unchanged`, the stat - /// cache of the snapshot that found `from`, it changes nothing if a - /// file it changes was edited since. + /// The changes that make the files of `from` those of `to`, after + /// fetching what the repository lacks of `to`. A move asks for them + /// before it publishes `moving`, so a fetch that fails moves nothing. + fn changes(&self, from: TreeId, to: TreeId) -> Result, WorkspaceError> { + self.need(&[to])?; + Ok(match self.store.diff(from, to) { + // A snapshot can lack trees its root names, if its fetch was + // cut short before fetches were marked: fetch it whole again. + Err(StoreError::NotFound(missing)) => match self.remote()? { + Some(remote) => { + self.sync(remote).repair(to)?; + self.store.diff(from, to)? + } + None => return Err(StoreError::NotFound(missing).into()), + }, + changes => changes?, + }) + } + + /// Makes the files match a snapshot, given `changes`, how they differ + /// from it, durably, cloning those the other workspaces have. With + /// `unchanged`, the stat cache of the snapshot `changes` start from, it + /// changes nothing if a file it changes was edited since. fn checkout( &self, - from: TreeId, - to: TreeId, + changes: &[FileChange], unchanged: Option<&StatCache>, ) -> Result<(), WorkspaceError> { - let changes = self.store.diff(from, to)?; if changes.is_empty() { return Ok(()); } @@ -830,7 +865,7 @@ impl Workspace { CloneSource::saved(root, cache) }) .collect(); - Ok(Worktree::new(&self.root, &self.store).checkout(&changes, &sources, unchanged)?) + Ok(Worktree::new(&self.root, &self.store).checkout(changes, &sources, unchanged)?) } /// `origin status`: prints the working copy's change, what it is on @@ -909,7 +944,7 @@ impl Workspace { let files = files.unwrap_or(revision.snapshot); println!( "Change {}{}", - revision.change.short(), + revision.change, self.flags(index, revision.change)? ); let edited = if files == revision.snapshot { @@ -939,6 +974,7 @@ impl Workspace { /// Prints the changes `snapshot` makes to `base`, line by line. fn print_diff(&self, base: Base, snapshot: TreeId) -> Result<(), WorkspaceError> { let base = self.base_tree(base)?; + self.need(&[snapshot])?; for change in self.store.diff(base, snapshot)? { print!("{}", FileDiff::new(&self.store, &change)?); } @@ -1156,6 +1192,11 @@ impl Workspace { heads[heads.len() - 1] }; let kept_revision: Revision = self.store.get(kept)?; + let snapshots = heads + .iter() + .map(|&head| Ok(self.store.get(head)?.snapshot)) + .collect::, WorkspaceError>>()?; + self.need(&snapshots)?; let base = self.merge_base(&heads)?; let mut snapshot = kept_revision.snapshot; let mut merged = Conflicts::default(); @@ -1275,7 +1316,12 @@ impl Workspace { (Some(limit), Some(head)) => (head.seq + 1).saturating_sub(limit as u64), _ => 0, }; - let log = self.service(None)?.log(&Caller::Owner, &target, since)?; + let query = LogQuery { + target: target.clone(), + since, + limit: None, + }; + let log = self.service(None)?.log(&Caller::Owner, &query)?; if target.id() == on { for (id, revision) in &stack { let marker = if *id == working_copy.id { '@' } else { '○' }; @@ -1445,12 +1491,15 @@ impl Workspace { shorts.join(", ") } - /// The snapshot a revision was made on top of. + /// The snapshot a revision was made on top of, fetched if the + /// repository lacks it. fn base_tree(&self, base: Base) -> Result { - Ok(match base { + let tree = match base { Base::Acceptance(id) => self.store.get(id)?.body().state, Base::Revision(id) => self.store.get(id)?.snapshot, - }) + }; + self.need(&[tree])?; + Ok(tree) } fn describe_base(&self, base: Base) -> Result { @@ -1506,6 +1555,15 @@ pub enum WorkspaceError { /// No target has this name. #[error("no target is named `{0}`")] UnknownTarget(TargetName), + /// No revision of the change is known. + #[error("change {} has no revisions", .0.short())] + UnknownChange(ChangeId), + /// A command needs a remote, and the repository has none. + #[error("this repository has no remote: set one with `origin remote `")] + NoRemote, + /// `clone` was given no path, and the URL names no directory. + #[error("cannot tell the repository's name from its URL: pass a path")] + Unnamed, /// An acceptance belongs to a target this repository doesn't have. #[error("no target has the id {0}")] UnknownTargetId(TargetId), @@ -1631,6 +1689,12 @@ pub enum WorkspaceError { /// The repository's service refused a call. #[error(transparent)] Service(#[from] ServiceError), + /// The remote cannot be opened, or refused a call. + #[error(transparent)] + Remote(#[from] RemoteError), + /// Syncing with the remote failed. + #[error(transparent)] + Sync(#[from] SyncError), /// `undo` found no operation of this workspace to take back. #[error("nothing to undo: no operation moved this workspace's working copy")] NothingToUndo, diff --git a/apps/origin-src/src/workspace/ops.rs b/apps/origin-src/src/workspace/ops.rs index 19b9df91..2dbb4139 100644 --- a/apps/origin-src/src/workspace/ops.rs +++ b/apps/origin-src/src/workspace/ops.rs @@ -207,33 +207,43 @@ impl Workspace { } /// `origin watch [--kind ]... [--subject ]... [--since - /// ]`: prints the events of those kinds about those subjects, - /// every event without them, as they happen, one a line, from `since`, - /// or from now. Runs until interrupted. + /// ] [--local]`: prints the events of those kinds about those + /// subjects, every event without them, as they happen, one a line, from + /// `since`, or from now. They are the remote's, long-polled, if the + /// repository has one, or with `local`, this node's. Runs until + /// interrupted. /// /// # Errors /// - /// Fails if the cursor expired, or the op log or a record cannot be read. + /// Fails if the cursor expired, the remote fails, or the op log or a + /// record cannot be read. pub fn watch( &self, kinds: Vec, subjects: Vec, since: Option, + local: bool, ) -> Result<(), WorkspaceError> { + let remote = if local { None } else { self.remote()? }; let service = self.service(None)?; let mut query = EventsQuery { cursor: since, + generation: None, wait: Some(Period(OriginService::MAX_WAIT)), limit: Some(256), kind: List(kinds), subject: List(subjects), }; loop { - let page = service.events(&Caller::Owner, &query)?; + let page = match remote { + Some(remote) => remote.events(&query)?, + None => service.events(&Caller::Owner, &query)?, + }; for event in &page.events { println!("{event}"); } query.cursor = Some(page.next); + query.generation = Some(page.generation); } } } diff --git a/apps/origin-src/src/workspace/registry.rs b/apps/origin-src/src/workspace/registry.rs index 201a0a6f..66bd3f8b 100644 --- a/apps/origin-src/src/workspace/registry.rs +++ b/apps/origin-src/src/workspace/registry.rs @@ -69,19 +69,12 @@ impl Workspace { let root = self.new_root(path, revision.change)?; self.write(&records)?; self.write_durably(self.registered(&root), &Self::path_bytes(&root))?; - let mut workspace = Self { - store: self - .store - .clone() - .with_actor(self.store.actor().in_workspace(&root)), - root, - config: self.config.clone(), - revision: id, - unlogged: None, - unreadable_move: None, - }; + let store = self.store.clone(); + let store = store.with_actor(self.store.actor().in_workspace(&root)); + let mut workspace = Self::at(root, store, self.config.clone(), id); let empty = Tree::default().encode().id().into(); - workspace.checkout(empty, revision.snapshot, None)?; + let changes = workspace.changes(empty, revision.snapshot)?; + workspace.checkout(&changes, None)?; let repository = Self::path_bytes(self.store.root()); workspace.write_durably(workspace.meta(Self::REPOSITORY), &repository)?; workspace.publish_pointer(None, id, Why::Command)?; diff --git a/apps/origin-src/src/workspace/remote.rs b/apps/origin-src/src/workspace/remote.rs new file mode 100644 index 00000000..274cfc1a --- /dev/null +++ b/apps/origin-src/src/workspace/remote.rs @@ -0,0 +1,380 @@ +//! The sync commands: `clone`, `remote`, `fetch`, `pull` and `push`, and +//! fetching the snapshots a lazy clone lacks. +//! +//! A repository has at most one remote, named in `.origin/remote`, which +//! its workspaces share. A clone holds the chains of the targets it +//! follows, the records of the changes it knows, and only the snapshots it +//! read: whenever a command needs one it lacks, such as a checkout, it is +//! fetched from the remote, skipping what the working copy already holds. +//! Syncing never changes files under a running workspace: `pull` is a +//! fetch, then a move of the working copy. +//! +//! The remote's event cursor as of the last fetch, and the generation of +//! the op log it's of, are kept in `.origin/remote-cursor`, so the next +//! fetch only compares the changes +//! that moved since (see [`Sync::moved`]). Every way this repository learns +//! of a change brings all the remote has of it, so the changes it knows are +//! whole as of that cursor. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::slice; +use std::sync::OnceLock; + +use origin_platform::Native; +use origin_shared::{ + Base, ChangeId, EncodedObject, Record, Revision, RevisionId, TargetName, Tree, TreeId, +}; + +use super::{Workspace, WorkspaceError}; +use crate::config::UserConfig; +use crate::index::Index; +use crate::oplog::{Actor, Why}; +use crate::remote::{Remote, RemoteUrl}; +use crate::store::Store; +use crate::sync::{Advanced, Cursor, Fetch, Sync}; + +impl Workspace { + /// The file in the repository's `.origin` naming its remote. + const REMOTE: &str = "remote"; + + /// `origin clone [] [--target ]`: creates a + /// repository in `path`, by default named after the remote's, which + /// follows `policy` and `target`, and starts a change on its tip. Only + /// that snapshot is fetched. + /// + /// # Errors + /// + /// Fails if `path` has files, the remote cannot be reached or has no + /// such target, or the repository cannot be written. A failed clone + /// leaves nothing behind, but the empty directory it was made in. + pub fn clone( + url: &str, + path: Option<&Path>, + target: &TargetName, + config: UserConfig, + actor: &Actor, + ) -> Result { + let url = RemoteUrl::parse(url)?; + let path = match path { + Some(path) => path.to_owned(), + None => PathBuf::from(url.name().ok_or(WorkspaceError::Unnamed)?), + }; + // Canonical, as `init` and every command opening it spell it. + let root = Self::canonical(&path)?; + if fs::read_dir(&root).is_ok_and(|mut entries| entries.next().is_some()) { + return Err(WorkspaceError::NotEmpty(root)); + } + // The topmost directory the clone creates, if `root` is missing: + // its parents may be too. Only where nothing is, not even a + // dangling symlink, which cleaning up would delete. + let created = root + .ancestors() + .take_while(|dir| dir.symlink_metadata().is_err()) + .last() + .map(Path::to_owned); + let remote = Remote::open(&url, Self::node_of(&config)?)?; + let cloned = Self::clone_into(root.clone(), &url, remote, target, config, actor); + if cloned.is_err() { + // Best effort: the error says what went wrong. + let _ = Self::undo_clone(&root, created.as_deref()); + } + cloned + } + + /// Removes what a failed clone into `root` left: the topmost directory + /// it `created`, with `root`, if it created any, or else everything in + /// `root`, which was empty, such as the files it checked out. + fn undo_clone(root: &Path, created: Option<&Path>) -> io::Result<()> { + if let Some(created) = created { + return fs::remove_dir_all(created); + } + for entry in fs::read_dir(root)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + fs::remove_dir_all(entry.path())?; + } else { + fs::remove_file(entry.path())?; + } + } + Ok(()) + } + + fn clone_into( + root: PathBuf, + url: &RemoteUrl, + remote: Remote, + target: &TargetName, + config: UserConfig, + actor: &Actor, + ) -> Result { + let meta = root.join(Self::META_DIR); + let store = Store::new(meta, Native::default()).with_actor(actor.in_workspace(&root)); + store.init()?; + let sync = Sync::new(&store, &remote, config.node().clone()); + let cursor = sync.moved(None)?.cursor; + sync.fetch_chains(slice::from_ref(target))?; + let tip = store + .head(target)? + .ok_or_else(|| WorkspaceError::UnknownTarget(target.clone()))?; + let state = store.get(tip.acceptance)?.body().state; + sync.need(state, &[])?; + + let empty_tree = Tree::default().encode(); + let empty_description = EncodedObject::blob(b""); + let revision = Revision { + change: ChangeId::random().map_err(WorkspaceError::Random)?, + predecessors: Vec::new(), + base: Base::Acceptance(tip.acceptance), + deps: Vec::new(), + snapshot: state, + description: empty_description.id().into(), + author: config.user(), + }; + let encoded = revision.encode(); + let id: RevisionId = encoded.id().into(); + let mut workspace = Self::at(root, store, config, id); + workspace.write(&[vec![empty_tree.clone(), empty_description], vec![encoded]])?; + workspace.write_durably( + workspace.store.root().join(Self::REMOTE), + format!("{url}\n").as_bytes(), + )?; + workspace.save_cursor(&cursor)?; + workspace.remote = OnceLock::from(Some(remote)); + let changes = workspace.changes(empty_tree.id().into(), state)?; + workspace.checkout(&changes, None)?; + workspace.publish_pointer(None, id, Why::Command)?; + println!( + "Cloned {url} into {}, following `{target}` at #{}", + workspace.root.display(), + tip.seq + ); + println!("Working copy: {}", workspace.headline(&revision)?); + println!("On top of: {}", workspace.describe_base(revision.base)?); + Ok(workspace) + } + + /// The file in the repository's `.origin` holding the remote's event + /// cursor as of the last fetch. + const CURSOR: &str = "remote-cursor"; + + /// The remote's event cursor as of the last fetch, if any, written + /// ` `. One that cannot be read is forgotten: the + /// next fetch compares every change. + fn cursor(&self) -> Option { + let path = self.store.root().join(Self::CURSOR); + let text = fs::read_to_string(path).ok()?; + let (generation, next) = text.trim_end().split_once(' ')?; + Some(Cursor { + generation: generation.to_owned(), + next: next.parse().ok()?, + }) + } + + /// Saves the remote's event cursor, without syncing: any older cursor, + /// or none, is as good, only slower to fetch from, and one torn by a + /// crash doesn't parse, or reads from earlier. + fn save_cursor(&self, cursor: &Cursor) -> Result<(), WorkspaceError> { + let path = self.store.root().join(Self::CURSOR); + let text = format!("{} {}\n", cursor.generation, cursor.next); + fs::write(&path, text).map_err(|source| WorkspaceError::io(&path, source)) + } + + /// Forgets the remote's event cursor: the next fetch compares every + /// change, for a new remote, or once the user sees more of it. + pub(super) fn forget_cursor(&self) -> Result<(), WorkspaceError> { + let path = self.store.root().join(Self::CURSOR); + match fs::remove_file(&path) { + Err(error) if error.kind() != io::ErrorKind::NotFound => { + Err(WorkspaceError::io(&path, error)) + } + _ => Ok(()), + } + } + + /// The URL of the repository's remote, if it has one. + fn remote_url(&self) -> Result, WorkspaceError> { + let path = self.store.root().join(Self::REMOTE); + match fs::read_to_string(&path) { + Ok(url) => Ok(Some(RemoteUrl::parse(url.trim_end())?)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(WorkspaceError::io(&path, source)), + } + } + + /// The repository's remote, opened the first time it's needed, if it + /// has one. + pub(super) fn remote(&self) -> Result, WorkspaceError> { + if let Some(remote) = self.remote.get() { + return Ok(remote.as_ref()); + } + let remote = match self.remote_url()? { + Some(url) => Some(Remote::open(&url, self.node()?)?), + None => None, + }; + Ok(self.remote.get_or_init(|| remote).as_ref()) + } + + /// Syncs with `remote`, as this node. + pub(super) fn sync<'s>(&'s self, remote: &'s Remote) -> Sync<'s> { + Sync::new(&self.store, remote, self.config.node().clone()) + } + + /// The repository's remote, which the command needs. + fn require_remote(&self) -> Result<&Remote, WorkspaceError> { + self.remote()?.ok_or(WorkspaceError::NoRemote) + } + + /// `origin remote []`: prints the repository's remote, or sets it. + /// + /// # Errors + /// + /// Fails if the URL is invalid, or the file naming the remote cannot be + /// read or written. + pub fn set_remote(&self, url: Option<&str>) -> Result<(), WorkspaceError> { + let Some(url) = url else { + match self.remote_url()? { + Some(url) => println!("{url}"), + None => println!("No remote: set one with `origin remote `."), + } + return Ok(()); + }; + let url = RemoteUrl::parse(url)?; + self.forget_cursor()?; + self.write_durably( + self.store.root().join(Self::REMOTE), + format!("{url}\n").as_bytes(), + )?; + println!("Remote: {url}"); + Ok(()) + } + + /// `origin fetch [--target ]... [--change ]...`: + /// fetches the chains of the targets this repository follows and of + /// `targets`, then the records of `changes` and of the changes it knows + /// that moved since the last fetch (see [`Sync::fetch`]). Changes + /// nothing under the working copy. + /// + /// # Errors + /// + /// Fails if there's no remote, it lacks a target or change asked for, + /// or the remote or the store fails. + pub fn fetch( + &self, + targets: &[TargetName], + changes: &[ChangeId], + ) -> Result<(), WorkspaceError> { + let remote = self.require_remote()?; + let since = self.cursor(); + let Fetch { + advanced, + fetched, + cursor, + } = self.sync(remote).fetch(targets, changes, since.as_ref())?; + if since.as_ref() != Some(&cursor) { + self.save_cursor(&cursor)?; + } + if !changes.is_empty() { + let index = Index::open(&self.store)?; + if let Some(&missing) = changes + .iter() + .find(|&&change| index.revisions(change).is_empty()) + { + return Err(WorkspaceError::UnknownChange(missing)); + } + } + for Advanced { target, from, to } in &advanced { + match from { + Some(from) => println!("`{target}`: #{} -> #{}", from.seq, to.seq), + None => println!("`{target}`: now followed, at #{}", to.seq), + } + } + if fetched.revisions + fetched.statements > 0 { + println!( + "Fetched {} revisions and {} statements", + fetched.revisions, fetched.statements + ); + } + if advanced.is_empty() && fetched.revisions + fetched.statements == 0 { + println!("Already up to date."); + } + Ok(()) + } + + /// `origin push []...`: shares the changes, the working copy's + /// by default, with the ones below them, with the remote: their + /// revisions, snapshots and statements. Lands nothing. + /// + /// # Errors + /// + /// Fails if there's no remote, a change cannot be found, is stale or is + /// on a target the remote lacks, or the remote or the store fails. + pub fn push(&mut self, changes: &[String]) -> Result<(), WorkspaceError> { + let working_copy = self.refresh()?; + let changes = if changes.is_empty() { + vec![working_copy.revision.change] + } else { + changes + .iter() + .map(|change| working_copy.find_change(change)) + .collect::>()? + }; + let pushed = self.sync(self.require_remote()?).push(&changes)?; + for statement in &pushed.unsent { + println!( + "Statement {} stays here: it needs a statement the remote doesn't have", + statement.short() + ); + } + if pushed.objects == 0 { + println!("Everything up to date."); + } else { + println!( + "Pushed {} revisions and {} statements ({} objects)", + pushed.revisions, pushed.statements, pushed.objects + ); + } + Ok(()) + } + + /// Fetches the snapshots of `trees` the repository lacks, if it has a + /// remote: a clone only holds the snapshots it read. Skips what the + /// working copy's revision and its base hold. Without a remote, reading + /// a missing snapshot fails later. + pub(super) fn need(&self, trees: &[TreeId]) -> Result<(), WorkspaceError> { + if self.store.held_whole(trees.iter().copied())?.len() == trees.len() { + return Ok(()); + } + let Some(remote) = self.remote()? else { + return Ok(()); + }; + let current = self.store.get(self.revision)?; + let base = match current.base { + Base::Acceptance(id) => self.store.get(id)?.body().state, + Base::Revision(id) => self.store.get(id)?.snapshot, + }; + let sync = self.sync(remote); + for &tree in trees { + sync.need(tree, &[current.snapshot, base])?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_failed_clone_empties_the_directory_it_found_empty() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join(".origin/objects")).unwrap(); + fs::create_dir(dir.path().join("src")).unwrap(); + fs::write(dir.path().join("src/lib.rs"), "fn lib() {}\n").unwrap(); + fs::write(dir.path().join("README.md"), "hello\n").unwrap(); + Workspace::undo_clone(dir.path(), None).unwrap(); + assert!(dir.path().is_dir()); + assert!(fs::read_dir(dir.path()).unwrap().next().is_none()); + } +} diff --git a/apps/origin-src/tests/cli.rs b/apps/origin-src/tests/cli.rs index 25547fda..516dc383 100644 --- a/apps/origin-src/tests/cli.rs +++ b/apps/origin-src/tests/cli.rs @@ -1,145 +1,24 @@ //! End-to-end tests of the change workflow and workspaces, driving the real //! `origin` binary. -use std::fs::{self, OpenOptions}; -use std::io::{BufRead, BufReader, Write}; +mod common; + +use std::fs; +use std::io::Write; use std::os::unix::fs::{MetadataExt, PermissionsExt}; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Output, Stdio}; +use std::process::{Command, Stdio}; use std::slice; use std::sync::mpsc; use std::thread; -use std::time::{Duration, Instant}; +use std::time::Duration; +use common::Repo; use origin_platform::Native; use origin_shared::{ ChangeId, EncodedObject, Kind, Record, Revision, RevisionId, Statement, StatementId, Verdict, }; use origin_src::store::Store; -/// A temporary working copy, and a user configuration of its own. -struct Repo { - dir: tempfile::TempDir, - config: tempfile::TempDir, -} - -impl Repo { - fn new() -> Self { - let config = tempfile::tempdir().unwrap(); - fs::write(config.path().join("config.toml"), "user = \"ada\"\n").unwrap(); - let repo = Self { - dir: tempfile::tempdir().unwrap(), - config, - }; - repo.ok(&["init"]); - repo - } - - /// `origin` in `cwd` as the configured user. - fn command(&self, cwd: &Path, args: &[&str]) -> Command { - let mut command = Command::new(env!("CARGO_BIN_EXE_origin")); - command - .args(args) - .current_dir(cwd) - .env("ORIGIN_CONFIG_DIR", self.config.path()); - command - } - - /// Runs `origin` in `cwd` as the configured user. - fn run(&self, cwd: &Path, args: &[&str]) -> Output { - self.command(cwd, args).output().unwrap() - } - - /// Starts a long-running command, such as `watch`, returning it and its - /// output lines as they come. - fn spawn(&self, args: &[&str]) -> (Child, mpsc::Receiver) { - let mut child = self - .command(self.dir.path(), args) - .stdout(Stdio::piped()) - .spawn() - .unwrap(); - let stdout = BufReader::new(child.stdout.take().unwrap()); - let (send, receive) = mpsc::channel(); - thread::spawn(move || { - for line in stdout.lines() { - if send.send(line.unwrap()).is_err() { - break; - } - } - }); - (child, receive) - } - - /// The revision the working copy is, in full. - fn revision(&self) -> String { - self.read(".origin/workspace").trim_end().to_owned() - } - - /// Runs a command that must succeed, returning its output. - fn ok(&self, args: &[&str]) -> String { - self.ok_in(self.dir.path(), args) - } - - /// Runs a command in `cwd` that must succeed, returning its output. - fn ok_in(&self, cwd: &Path, args: &[&str]) -> String { - let output = self.run(cwd, args); - assert!( - output.status.success(), - "`origin {}` failed: {}", - args.join(" "), - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).unwrap() - } - - /// Runs a command that must fail, returning its error. - fn fails(&self, args: &[&str]) -> String { - self.fails_in(self.dir.path(), args) - } - - /// Runs `origin` in `cwd`, which must fail, returning its errors. - fn fails_in(&self, cwd: &Path, args: &[&str]) -> String { - let output = self.run(cwd, args); - assert!( - !output.status.success(), - "`origin {}` succeeded", - args.join(" ") - ); - String::from_utf8(output.stderr).unwrap() - } - - fn path(&self, path: &str) -> std::path::PathBuf { - self.dir.path().join(path) - } - - fn write(&self, path: &str, content: &str) { - let path = self.path(path); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(path, content).unwrap(); - } - - fn read(&self, path: &str) -> String { - fs::read_to_string(self.path(path)).unwrap() - } - - /// The short id of the working copy's change. - fn change(&self) -> String { - let status = self.ok(&["status"]); - let line = status.lines().next().unwrap(); - let headline = line.strip_prefix("Working copy: ").unwrap(); - headline.split(' ').next().unwrap().to_owned() - } - - /// Creates a workspace at `path`, returning its change. - fn workspace(&self, path: &Path, args: &[&str]) -> String { - let path = path.to_str().unwrap(); - let output = self.ok(&[&["workspace", "new", path], args].concat()); - let line = output.lines().nth(1).unwrap(); - let headline = line.strip_prefix("Working copy: ").unwrap(); - headline.split(' ').next().unwrap().to_owned() - } -} - #[test] fn the_working_copy_is_a_revision_of_the_current_change() { let repo = Repo::new(); @@ -597,8 +476,8 @@ fn a_statement_takes_its_details_as_text_or_from_stdin() { let mut piped = Command::new(env!("CARGO_BIN_EXE_origin")) .args(["annotate", "@", "ci.test", "--fail", "--payload", "-"]) - .current_dir(repo.dir.path()) - .env("ORIGIN_CONFIG_DIR", repo.config.path()) + .current_dir(repo.root()) + .env("ORIGIN_CONFIG_DIR", repo.config()) .stdin(Stdio::piped()) .spawn() .unwrap(); @@ -631,7 +510,7 @@ fn a_statement_takes_its_details_as_text_or_from_stdin() { #[test] fn init_with_a_malformed_key_leaves_no_repository() { let repo = Repo::new(); - fs::write(repo.config.path().join("node.key"), b"short").unwrap(); + fs::write(repo.config().join("node.key"), b"short").unwrap(); let other = tempfile::tempdir().unwrap(); assert!(!repo.run(other.path(), &["init"]).status.success()); assert!(!other.path().join(".origin").exists()); @@ -644,7 +523,7 @@ fn only_the_creator_of_a_change_abandons_it() { repo.ok(&["describe", "Ada's change"]); let change = repo.change(); // Eve takes the change over, then tries to give it up. - let config = repo.config.path().join("config.toml"); + let config = repo.config().join("config.toml"); fs::write(&config, "user = \"eve\"\n").unwrap(); repo.write("file", "eve's\n"); repo.ok(&["describe", "Eve's now"]); @@ -675,7 +554,7 @@ fn init_creates_policy_and_main_with_one_node_key() { ); assert!(repo.fails(&["init"]).contains("already has a repository")); - let key = repo.config.path().join("node.key"); + let key = repo.config().join("node.key"); let mode = fs::metadata(&key).unwrap().permissions().mode(); assert_eq!(mode & 0o777, 0o600); let before = fs::read(&key).unwrap(); @@ -725,10 +604,10 @@ fn a_closed_output_pipe_ends_quietly() { let repo = Repo::new(); let (reader, writer) = std::io::pipe().unwrap(); drop(reader); - let output = Command::new(env!("CARGO_BIN_EXE_origin")) + let output = std::process::Command::new(env!("CARGO_BIN_EXE_origin")) .arg("status") - .current_dir(repo.dir.path()) - .env("ORIGIN_CONFIG_DIR", repo.config.path()) + .current_dir(repo.root()) + .env("ORIGIN_CONFIG_DIR", repo.config()) .stdout(writer) .output() .unwrap(); @@ -929,7 +808,7 @@ fn an_argument_that_is_not_utf8_is_no_panic() { let repo = Repo::new(); let path = std::ffi::OsStr::from_bytes(b"gone\xff"); let output = repo - .command(repo.dir.path(), &["workspace", "rm"]) + .command(repo.root(), &["workspace", "rm"]) .arg(path) .output() .unwrap(); @@ -976,7 +855,7 @@ fn a_move_marker_that_cannot_be_read_is_kept_and_stops_moves() { let marker = repo.path(".origin/moving"); fs::create_dir(&marker).unwrap(); - let status = repo.run(repo.dir.path(), &["status"]); + let status = repo.run(repo.root(), &["status"]); let warning = String::from_utf8_lossy(&status.stderr); assert!(status.status.success(), "{warning}"); assert!(warning.starts_with("warning: cannot read "), "{warning}"); @@ -1834,7 +1713,7 @@ fn undo_moves_only_its_own_workspace() { "{log}" ); assert!(headers[1].starts_with("@ #") && headers[1].ends_with(": origin new Main work")); - assert!(headers[1].contains(&repo.dir.path().display().to_string())); + assert!(headers[1].contains(&repo.root().display().to_string())); assert!(log.contains(" undid #"), "{log}"); assert_eq!( repo.ok(&["op", "log", "-n", "1"]).lines().count(), @@ -1843,56 +1722,6 @@ fn undo_moves_only_its_own_workspace() { ); } -impl Repo { - /// The file the store keeps the object holding `content` in. - fn stored_object(&self, content: &[u8]) -> PathBuf { - let hex = origin_shared::EncodedObject::blob(content).id().to_hex(); - self.path(&format!( - ".origin/objects/loose/{}/{}", - &hex[..2], - &hex[2..] - )) - } - - /// Runs `origin ` in `cwd` until it reads the stored object - /// holding `content`, then once `ready`, kills it, as a crash would, - /// and stores the object again. - fn crash_reading(&self, cwd: &Path, content: &[u8], args: &[&str], ready: impl Fn() -> bool) { - let stored = self.stored_object(content); - let object = fs::read(&stored).unwrap(); - fs::remove_file(&stored).unwrap(); - let fifo = Command::new("mkfifo").arg(&stored).status().unwrap(); - assert!(fifo.success()); - let mut child = self - .command(cwd, args) - .stdout(Stdio::null()) - .spawn() - .unwrap(); - // Opening the pipe to write waits for the command to open it to - // read, which it then waits on, for as long as the pipe is open. - let (opened, open) = mpsc::channel(); - let pipe = stored.clone(); - thread::spawn(move || opened.send(OpenOptions::new().write(true).open(pipe))); - let deadline = Instant::now() + Duration::from_secs(20); - let _pipe = loop { - if let Ok(pipe) = open.recv_timeout(Duration::from_millis(20)) { - break pipe.unwrap(); - } - assert!(child.try_wait().unwrap().is_none(), "it never read it"); - assert!(Instant::now() < deadline, "it never read it in time"); - }; - while !ready() { - assert!(Instant::now() < deadline, "never ready"); - thread::sleep(Duration::from_millis(20)); - } - child.kill().unwrap(); - child.wait().unwrap(); - fs::remove_file(&stored).unwrap(); - fs::write(&stored, object).unwrap(); - fs::set_permissions(&stored, fs::Permissions::from_mode(0o444)).unwrap(); - } -} - #[test] fn a_move_cut_short_is_finished_without_losing_edits() { let repo = Repo::new(); @@ -1907,7 +1736,7 @@ fn a_move_cut_short_is_finished_without_losing_edits() { fs::remove_file(repo.path("ro/b")).unwrap(); // `edit` back to the first change crashes after writing `a`. - repo.crash_reading(repo.dir.path(), b"b\n", &["edit", &first], || { + repo.crash_reading(repo.root(), b"b\n", &["edit", &first], || { fs::read_to_string(repo.path("a")).is_ok_and(|a| a == "a\n") }); assert!(repo.path(".origin/moving").exists()); @@ -2047,7 +1876,7 @@ fn a_move_that_cannot_be_finished_leaves_the_files() { // `edit` crashes after writing `a`, and the content finishing it // needs can't be read. - repo.crash_reading(repo.dir.path(), b"b\n", &["edit", &first], || { + repo.crash_reading(repo.root(), b"b\n", &["edit", &first], || { fs::read_to_string(repo.path("a")).is_ok_and(|a| a == "a\n") }); assert!(repo.path(".origin/moving").exists()); diff --git a/apps/origin-src/tests/common/mod.rs b/apps/origin-src/tests/common/mod.rs new file mode 100644 index 00000000..4bbb0063 --- /dev/null +++ b/apps/origin-src/tests/common/mod.rs @@ -0,0 +1,328 @@ +//! Helpers for the end-to-end tests, which run the real `origin` binary. + +// Each test binary uses its own subset of the helpers, and the others would +// warn as unused in it. +#![allow(dead_code)] + +use std::fs::{self, OpenOptions}; +use std::io::{BufRead, BufReader}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Output, Stdio}; +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +/// A user, with a configuration directory, and so a node key, of their +/// own. Clones share it. +#[derive(Clone)] +pub struct User { + config: Arc, +} + +impl User { + /// A user named `name`, whose node is named after them. + pub fn new(name: &str) -> Self { + let config = tempfile::tempdir().unwrap(); + fs::write( + config.path().join("config.toml"), + format!("user = \"{name}\"\n"), + ) + .unwrap(); + Self { + config: Arc::new(config), + } + } + + /// The configuration directory. + pub fn config(&self) -> &Path { + self.config.path() + } + + /// `origin` with `args`, run in `cwd` as this user. + pub fn command(&self, cwd: &Path, args: &[&str]) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_origin")); + command + .args(args) + .current_dir(cwd) + .env("ORIGIN_CONFIG_DIR", self.config()); + command + } +} + +/// A temporary working copy, and its user. +pub struct Repo { + dir: tempfile::TempDir, + user: User, +} + +impl Repo { + /// A new repository of a user of its own, `ada`. + pub fn new() -> Self { + Self::init(&User::new("ada")) + } + + /// A new repository of `user`. + pub fn init(user: &User) -> Self { + let repo = Self::empty(user); + repo.ok(&["init"]); + repo + } + + /// A clone of `url` by `user`, with more arguments for `clone`. + pub fn clone(user: &User, url: &str, args: &[&str]) -> Self { + let repo = Self::empty(user); + let root = repo.root().to_str().unwrap().to_owned(); + repo.ok(&[&["clone", url, &root], args].concat()); + repo + } + + fn empty(user: &User) -> Self { + Self { + dir: tempfile::tempdir().unwrap(), + user: user.clone(), + } + } + + /// The root of the working copy. + pub fn root(&self) -> &Path { + self.dir.path() + } + + /// The user's configuration directory. + pub fn config(&self) -> &Path { + self.user.config() + } + + /// Runs `origin` in `cwd` as the repository's user. + pub fn run(&self, cwd: &Path, args: &[&str]) -> Output { + self.command(cwd, args).output().unwrap() + } + + /// `origin` with `args`, to run in `cwd` as this repository's user. + pub fn command(&self, cwd: &Path, args: &[&str]) -> Command { + self.user.command(cwd, args) + } + + /// Starts a long-running command, such as `watch`, returning it and its + /// output lines as they come. + pub fn spawn(&self, args: &[&str]) -> (Child, mpsc::Receiver) { + let mut child = self + .user + .command(self.root(), args) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let stdout = BufReader::new(child.stdout.take().unwrap()); + let (send, receive) = mpsc::channel(); + thread::spawn(move || { + for line in stdout.lines() { + if send.send(line.unwrap()).is_err() { + break; + } + } + }); + (child, receive) + } + + /// The revision the working copy is, in full. + pub fn revision(&self) -> String { + self.read(".origin/workspace").trim_end().to_owned() + } + + /// Runs a command that must succeed, returning its output. + pub fn ok(&self, args: &[&str]) -> String { + self.ok_in(self.root(), args) + } + + /// Runs a command in `cwd` that must succeed, returning its output. + pub fn ok_in(&self, cwd: &Path, args: &[&str]) -> String { + let output = self.run(cwd, args); + assert!( + output.status.success(), + "`origin {}` failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + } + + /// Runs a command that must fail, returning its error. + pub fn fails(&self, args: &[&str]) -> String { + self.fails_in(self.root(), args) + } + + /// Runs `origin` in `cwd`, which must fail, returning its errors. + pub fn fails_in(&self, cwd: &Path, args: &[&str]) -> String { + let output = self.run(cwd, args); + assert!( + !output.status.success(), + "`origin {}` succeeded", + args.join(" ") + ); + String::from_utf8(output.stderr).unwrap() + } + + pub fn path(&self, path: &str) -> PathBuf { + self.root().join(path) + } + + pub fn write(&self, path: &str, content: &str) { + let path = self.path(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); + } + + pub fn read(&self, path: &str) -> String { + fs::read_to_string(self.path(path)).unwrap() + } + + /// The short id of the working copy's change. + pub fn change(&self) -> String { + let status = self.ok(&["status"]); + let line = status.lines().next().unwrap(); + let headline = line.strip_prefix("Working copy: ").unwrap(); + headline.split(' ').next().unwrap().to_owned() + } + + /// The full id of the working copy's change. + pub fn full_change(&self) -> String { + let show = self.ok(&["show"]); + let line = show.lines().next().unwrap(); + let change = line.strip_prefix("Change ").unwrap(); + change.split(' ').next().unwrap().to_owned() + } + + /// Creates a workspace at `path`, returning its change. + pub fn workspace(&self, path: &Path, args: &[&str]) -> String { + let path = path.to_str().unwrap(); + let output = self.ok(&[&["workspace", "new", path], args].concat()); + let line = output.lines().nth(1).unwrap(); + let headline = line.strip_prefix("Working copy: ").unwrap(); + headline.split(' ').next().unwrap().to_owned() + } + + /// The file the store keeps the object holding `content` in. + pub fn stored_object(&self, content: &[u8]) -> PathBuf { + let hex = origin_shared::EncodedObject::blob(content).id().to_hex(); + self.path(&format!( + ".origin/objects/loose/{}/{}", + &hex[..2], + &hex[2..] + )) + } + + /// Runs `origin ` in `cwd` until it reads the stored object + /// holding `content`, then once `ready`, kills it, as a crash would, + /// and stores the object again. + pub fn crash_reading( + &self, + cwd: &Path, + content: &[u8], + args: &[&str], + ready: impl Fn() -> bool, + ) { + let stored = self.stored_object(content); + let object = fs::read(&stored).unwrap(); + fs::remove_file(&stored).unwrap(); + let fifo = Command::new("mkfifo").arg(&stored).status().unwrap(); + assert!(fifo.success()); + let mut child = self + .user + .command(cwd, args) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + // Opening the pipe to write waits for the command to open it to + // read, which it then waits on, for as long as the pipe is open. + let (opened, open) = mpsc::channel(); + let pipe = stored.clone(); + thread::spawn(move || opened.send(OpenOptions::new().write(true).open(pipe))); + let deadline = Instant::now() + Duration::from_secs(20); + let _pipe = loop { + if let Ok(pipe) = open.recv_timeout(Duration::from_millis(20)) { + break pipe.unwrap(); + } + assert!(child.try_wait().unwrap().is_none(), "it never read it"); + assert!(Instant::now() < deadline, "it never read it in time"); + }; + while !ready() { + assert!(Instant::now() < deadline, "never ready"); + thread::sleep(Duration::from_millis(20)); + } + child.kill().unwrap(); + child.wait().unwrap(); + fs::remove_file(&stored).unwrap(); + fs::write(&stored, object).unwrap(); + fs::set_permissions(&stored, fs::Permissions::from_mode(0o444)).unwrap(); + } +} + +/// A running `origin serve` hosting one repository, `demo`, which its node, +/// `hosted`, orders. Killed when dropped. +pub struct Server { + dir: tempfile::TempDir, + child: Child, + /// The server's base URL. + pub base: String, +} + +impl Server { + pub fn start() -> Self { + let dir = tempfile::tempdir().unwrap(); + let config = dir.path().join("origin-server.toml"); + fs::write( + &config, + "listen = \"127.0.0.1:0\"\nnode = \"hosted\"\n[repos]\ndemo = \"repos/demo\"\n", + ) + .unwrap(); + let origin = |args: &[&str]| { + let mut command = Command::new(env!("CARGO_BIN_EXE_origin")); + command.args(args).arg("--config").arg(&config); + command + }; + let created = origin(&["repo", "create"]) + .arg(dir.path().join("repos/demo")) + .output() + .unwrap(); + assert!( + created.status.success(), + "{}", + String::from_utf8_lossy(&created.stderr) + ); + let mut child = origin(&["serve"]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let mut line = String::new(); + BufReader::new(child.stdout.take().unwrap()) + .read_line(&mut line) + .unwrap(); + let base = line + .trim() + .strip_prefix("Listening on ") + .unwrap() + .to_owned(); + Self { dir, child, base } + } + + /// The directory holding the server's configuration, key and + /// repositories. + pub fn dir(&self) -> &Path { + self.dir.path() + } + + /// The URL of `demo`. + pub fn url(&self) -> String { + format!("{}/demo", self.base) + } +} + +impl Drop for Server { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/apps/origin-src/tests/http.rs b/apps/origin-src/tests/http.rs index 5a4c46f7..7c38d1c6 100644 --- a/apps/origin-src/tests/http.rs +++ b/apps/origin-src/tests/http.rs @@ -7,16 +7,16 @@ use std::time::{Duration, Instant}; use origin_client::{Client, Error, Repo}; use origin_shared::api::{ - ErrorCode, EventsQuery, FetchRecords, LandRequest, List, LogQuery, NewTarget, RepoName, - StatementsQuery, + ErrorCode, EventsQuery, Expected, FetchRecords, FetchSnapshot, InventoryRequest, LandRequest, + List, LogQuery, NewTarget, PutQuery, RepoName, StatementsQuery, }; use origin_shared::event::EventKind; use origin_shared::landing::{Blocked, Landing, Proposal}; use origin_shared::pack::Pack; use origin_shared::{ Base, BlobId, ChangeId, Digest, EncodedObject, EntryKind, EntryName, Inputs, Kind, NodeKey, - Period, Record, RecordId, Revision, RevisionId, Source, Statement, TargetName, Tree, TreeEntry, - Verdict, + Period, Record, RecordId, RepoPath, Revision, RevisionId, Source, Statement, TargetName, Tree, + TreeEntry, TreeId, Verdict, }; use origin_src::http::Api; use origin_src::policy::Policy; @@ -201,14 +201,29 @@ async fn puts_and_fetches_records() { "records are immutable, so putting them again does nothing" ); - let wanted = [made.objects[4].clone(), made.objects[0].clone()]; + let [a, _, tree, description, revision] = &made.objects[..] else { + unreachable!() + }; let fetched = repo .fetch_records(&FetchRecords { - ids: wanted.iter().map(EncodedObject::id).collect(), + ids: vec![revision.id()], }) .await .unwrap(); - assert_eq!(fetched.objects, wanted, "verified, in the order asked"); + assert_eq!( + fetched.pack.objects, + [revision.clone(), description.clone()], + "verified, with the description" + ); + for content in [a, tree] { + let error = repo + .fetch_records(&FetchRecords { + ids: vec![content.id()], + }) + .await + .unwrap_err(); + assert_eq!(code(error), ErrorCode::BadRequest, "content leaves by path"); + } // A record is refused unless the repository can use it. let orphan = Made::revision(change(2), main_tip(repo).await, vec![], &[("c.txt", "c")]); @@ -221,6 +236,7 @@ async fn puts_and_fetches_records() { .log(&LogQuery { target: TargetName::MAIN, since: 0, + limit: None, }) .await .unwrap(); @@ -307,6 +323,43 @@ async fn puts_refuse_objects_named_as_another_kind() { assert_eq!(refused(vec![tree.encode()]).await, ErrorCode::BadRequest); } +#[tokio::test] +async fn pushes_compare_and_swap_the_heads_they_saw() { + let server = Server::start().await; + let repo = &server.demo; + let base = main_tip(repo).await; + let expect = |heads: &[RevisionId]| PutQuery { + expect: List(vec![Expected { + change: change(1), + heads: heads.to_vec(), + }]), + }; + let first = Made::revision(change(1), base, Vec::new(), &[("a", "first")]); + repo.push_records(&first.pack(), &expect(&[])) + .await + .unwrap(); + + // Two pushes made from the same view of the change: the second is + // refused, and publishes no record. + let left = Made::revision(change(1), base, vec![first.id], &[("a", "left")]); + let right = Made::revision(change(1), base, vec![first.id], &[("a", "right")]); + repo.push_records(&left.pack(), &expect(&[first.id])) + .await + .unwrap(); + let error = repo + .push_records(&right.pack(), &expect(&[first.id])) + .await + .unwrap_err(); + assert_eq!(code(error), ErrorCode::Conflict); + let request = InventoryRequest { + changes: vec![change(1)], + }; + assert_eq!(repo.inventory(&request).await.unwrap()[0].heads, [left.id]); + repo.push_records(&right.pack(), &expect(&[left.id])) + .await + .unwrap(); +} + #[tokio::test] async fn promotions_must_match_their_original() { let server = Server::start().await; @@ -357,6 +410,68 @@ async fn promotions_must_match_their_original() { assert_eq!(stored.new, 1, "the original was stored with the forgery"); } +#[tokio::test] +async fn fetches_snapshots_by_walking_them() { + let server = Server::start().await; + let repo = &server.demo; + let made = Made::revision( + change(1), + main_tip(repo).await, + vec![], + &[("a.txt", "a"), ("b.txt", "b")], + ); + repo.put_records(&made.pack()).await.unwrap(); + let [a, b, tree, description, _] = &made.objects[..] else { + unreachable!() + }; + + let snapshot = |have: Vec, paths: &[&str]| FetchSnapshot { + snapshot: tree.id().into(), + have, + paths: paths + .iter() + .map(|path| RepoPath::new(path).unwrap()) + .collect(), + skip: 0, + }; + let whole = repo.fetch_snapshot(&snapshot(vec![], &[])).await.unwrap(); + assert_eq!(whole.pack.objects, [tree.clone(), a.clone(), b.clone()]); + let only_b = repo + .fetch_snapshot(&snapshot(vec![], &["b.txt"])) + .await + .unwrap(); + assert_eq!(only_b.pack.objects, [tree.clone(), b.clone()]); + let edited = Made::revision( + change(3), + main_tip(repo).await, + vec![], + &[("a.txt", "a"), ("b.txt", "c")], + ); + repo.put_records(&edited.pack()).await.unwrap(); + let changed = repo + .fetch_snapshot(&FetchSnapshot { + snapshot: edited.objects[2].id().into(), + have: vec![tree.id().into()], + paths: Vec::new(), + skip: 0, + }) + .await + .unwrap(); + assert_eq!( + changed.pack.objects, + [edited.objects[2].clone(), edited.objects[1].clone()], + "the new tree and the edited file" + ); + let missing = FetchSnapshot { + snapshot: description.id().into(), + have: Vec::new(), + paths: Vec::new(), + skip: 0, + }; + let error = repo.fetch_snapshot(&missing).await.unwrap_err(); + assert_eq!(code(error), ErrorCode::NotFound, "a blob is no snapshot"); +} + #[tokio::test] async fn reads_statements_and_revisions() { let server = Server::start().await; @@ -401,6 +516,24 @@ async fn reads_statements_and_revisions() { let error = repo.revisions(change(9)).await.unwrap_err(); assert_eq!(code(error), ErrorCode::NotFound); + + let inventories = repo + .inventory(&InventoryRequest { + changes: vec![change(1), change(9)], + }) + .await + .unwrap(); + let [known, unknown] = &inventories[..] else { + panic!("{inventories:?}") + }; + assert_eq!(known.revisions, ids); + assert_eq!(known.heads, [left.id, right.id]); + let mut statements = known.statements.clone(); + statements.sort(); + let mut expected = vec![statement.id().into(), flagged[0].id]; + expected.sort(); + assert_eq!(statements, expected); + assert!(unknown.revisions.is_empty() && unknown.statements.is_empty()); } #[tokio::test] @@ -435,6 +568,7 @@ async fn lands_changes_and_targets() { .log(&LogQuery { target: TargetName::MAIN, since: 1, + limit: Some(1), }) .await .unwrap(); diff --git a/apps/origin-src/tests/serve.rs b/apps/origin-src/tests/serve.rs index 0ad82b58..0e10f73d 100644 --- a/apps/origin-src/tests/serve.rs +++ b/apps/origin-src/tests/serve.rs @@ -1,58 +1,18 @@ //! End to end: `origin repo create` and `origin serve`, running the real //! binary and talking to it through the SDK. -use std::fs; -use std::io::{BufRead, BufReader}; -use std::process::{Child, Command, Stdio}; +mod common; +use common::Server; use origin_client::Client; use origin_shared::api::RepoName; -/// A running `origin serve`, killed when dropped. -struct Serve(Child); - -impl Drop for Serve { - fn drop(&mut self) { - let _ = self.0.kill(); - let _ = self.0.wait(); - } -} - #[tokio::test] async fn serves_the_repositories_it_created() { - let dir = tempfile::tempdir().unwrap(); - let config = dir.path().join("origin-server.toml"); - fs::write( - &config, - "listen = \"127.0.0.1:0\"\nnode = \"hosted\"\n[repos]\ndemo = \"repos/demo\"\n", - ) - .unwrap(); - let origin = |args: &[&str]| { - let mut command = Command::new(env!("CARGO_BIN_EXE_origin")); - command.args(args).arg("--config").arg(&config); - command - }; - - let created = origin(&["repo", "create"]) - .arg(dir.path().join("repos/demo")) - .output() - .unwrap(); - assert!( - created.status.success(), - "{}", - String::from_utf8_lossy(&created.stderr) - ); - assert!(dir.path().join("node.key").is_file(), "the server's key"); - - let mut child = origin(&["serve"]).stdout(Stdio::piped()).spawn().unwrap(); - let mut line = String::new(); - BufReader::new(child.stdout.take().unwrap()) - .read_line(&mut line) - .unwrap(); - let _serve = Serve(child); - let url = line.trim().strip_prefix("Listening on ").unwrap(); + let server = Server::start(); + assert!(server.dir().join("node.key").is_file(), "the server's key"); - let client = Client::new(url); + let client = Client::new(&server.base); let repos = client.repos().await.unwrap(); assert_eq!(repos.len(), 1); let targets = client diff --git a/apps/origin-src/tests/sync.rs b/apps/origin-src/tests/sync.rs new file mode 100644 index 00000000..99572a34 --- /dev/null +++ b/apps/origin-src/tests/sync.rs @@ -0,0 +1,224 @@ +//! End-to-end tests of sharing work: cloning, pushing, fetching and +//! pulling, and landing and watching against a remote, over HTTP and +//! through a local path. + +mod common; + +use std::fs; +use std::time::{Duration, Instant}; + +use common::{Repo, Server, User}; + +#[test] +fn work_lands_through_the_server_and_reaches_other_clones() { + let server = Server::start(); + let (ada, bob) = (User::new("ada"), User::new("bob")); + let alice = Repo::clone(&ada, &server.url(), &[]); + assert!( + alice + .ok(&["status"]) + .ends_with("On top of: main #0\nNo changes.\n") + ); + alice.write("README.md", "hello\n"); + alice.write("src/lib.rs", "fn lib() {}\n"); + alice.ok(&["describe", "Add the library"]); + let change = alice.change(); + assert!(alice.ok(&["push"]).starts_with("Pushed ")); + assert_eq!(alice.ok(&["push"]), "Everything up to date.\n"); + // The server orders `main`, so the landing goes to its queue. + let landed = alice.ok(&["land"]); + assert!( + landed.contains(&format!("Landed change {change}")), + "{landed}" + ); + assert!(landed.contains("on main #1"), "{landed}"); + + let bobs = Repo::clone(&bob, &server.url(), &[]); + assert_eq!(bobs.read("README.md"), "hello\n"); + assert_eq!(bobs.read("src/lib.rs"), "fn lib() {}\n"); + let log = bobs.ok(&["log"]); + assert!(log.contains(&format!("{change} Add the library")), "{log}"); + + // Bob works on top while Alice lands more: pulling moves his work onto + // the new tip. + bobs.write("notes.md", "bob\n"); + alice.write("README.md", "hello, world\n"); + alice.ok(&["land"]); + let pulled = bobs.ok(&["pull"]); + assert!(pulled.starts_with("`main`: #1 -> #2\n"), "{pulled}"); + assert_eq!(bobs.read("README.md"), "hello, world\n"); + assert_eq!(bobs.read("notes.md"), "bob\n"); + let status = bobs.ok(&["status"]); + assert!( + status.contains("On top of: main #2\nA notes.md\n"), + "{status}" + ); + assert!(bobs.ok(&["pull"]).ends_with("on the tip of `main`.\n")); +} + +#[test] +fn pushing_a_change_that_moved_on_the_server_is_refused() { + let server = Server::start(); + let (ada, bob) = (User::new("ada"), User::new("bob")); + let alice = Repo::clone(&ada, &server.url(), &[]); + alice.write("file", "alice\n"); + alice.ok(&["describe", "Shared work"]); + alice.ok(&["push"]); + + // Bob takes the change over, and pushes a revision of it. + let bobs = Repo::clone(&bob, &server.url(), &[]); + let change = alice.full_change(); + bobs.ok(&["fetch", "--change", &change]); + bobs.ok(&["edit", &change]); + assert_eq!(bobs.read("file"), "alice\n"); + bobs.write("file", "bob\n"); + bobs.ok(&["push", &change]); + + // Alice's next revision is stale: she learns before sending anything. + alice.write("file", "alice again\n"); + let error = alice.fails(&["push"]); + assert!(error.contains("is stale"), "{error}"); + alice.ok(&["fetch"]); + assert!(alice.ok(&["status"]).contains("Divergent")); + alice.ok(&["converge", &change]); + assert!(alice.ok(&["push"]).starts_with("Pushed ")); +} + +#[test] +fn watch_follows_the_remote_events() { + let server = Server::start(); + let ada = User::new("ada"); + let alice = Repo::clone(&ada, &server.url(), &[]); + let watcher = Repo::clone(&ada, &server.url(), &[]); + // From the first event, so the landing shows whenever the watch starts. + let (mut watch, lines) = watcher.spawn(&["watch", "--kind", "target.advanced", "--since", "0"]); + alice.write("file", "content\n"); + alice.ok(&["land"]); + let deadline = Instant::now() + Duration::from_secs(20); + loop { + let line = lines + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .expect("the landing on main within the deadline"); + if line.contains("target.advanced target=main seq=1") { + break; + } + } + watch.kill().unwrap(); + watch.wait().unwrap(); +} + +#[test] +fn clones_make_the_directories_they_go_in() { + let ada = User::new("ada"); + let origin = Repo::init(&ada); + let url = origin.root().to_str().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let cloned = ada + .command(dir.path(), &["clone", url, "x/y/name"]) + .output() + .unwrap(); + assert!(cloned.status.success(), "{cloned:?}"); + let root = fs::canonicalize(dir.path().join("x/y/name")).unwrap(); + let undo = ada.command(&root, &["undo"]).output().unwrap(); + let error = String::from_utf8_lossy(&undo.stderr); + assert!( + error.contains("nothing to undo but the creation of"), + "{error}" + ); + + // A clone that fails leaves none of them behind. + let args = ["clone", url, "z/y/name", "--target", "nope"]; + assert!(!ada.command(dir.path(), &args).status().unwrap().success()); + assert!(!dir.path().join("z").exists()); + + // Nor does it take a dangling symlink it was told to clone into. + let link = dir.path().join("link"); + std::os::unix::fs::symlink("nowhere", &link).unwrap(); + assert!( + !ada.command(dir.path(), &["clone", url, "link"]) + .status() + .unwrap() + .success() + ); + assert!(link.symlink_metadata().is_ok(), "the symlink stays"); +} + +#[test] +fn local_paths_are_remotes_served_in_process() { + let ada = User::new("ada"); + let origin = Repo::init(&ada); + origin.write("README.md", "hello\n"); + origin.ok(&["land"]); + + // Ada's node orders the repository, so her clone's queue, running in + // process, lands in it. + let url = origin.root().to_str().unwrap(); + let clone = Repo::clone(&ada, url, &[]); + assert_eq!(clone.read("README.md"), "hello\n"); + clone.write("src/lib.rs", "fn lib() {}\n"); + let landed = clone.ok(&["land"]); + assert!(landed.contains("on main #2"), "{landed}"); + let log = origin.ok(&["log", "main"]); + assert!(log.contains("◆ main #2"), "{log}"); + + // Bob's node doesn't: the queue refuses him. + let bobs = Repo::clone(&User::new("bob"), url, &[]); + assert_eq!(bobs.read("src/lib.rs"), "fn lib() {}\n"); + bobs.write("bob.txt", "bob\n"); + let error = bobs.fails(&["land"]); + assert!(error.contains("authority"), "{error}"); +} + +#[test] +fn clones_follow_one_target_and_fetch_others_on_demand() { + let ada = User::new("ada"); + let origin = Repo::init(&ada); + origin.write("README.md", "main\n"); + origin.ok(&["land"]); + origin.ok(&["target", "new", "release", "--from", "main"]); + origin.ok(&["new", "--on", "release"]); + origin.write("RELEASE.md", "v1\n"); + origin.ok(&["land"]); + + let url = origin.root().to_str().unwrap(); + let clone = Repo::clone(&User::new("bob"), url, &["--target", "release"]); + assert_eq!(clone.read("RELEASE.md"), "v1\n"); + assert!(clone.ok(&["log"]).contains("◆ release #1")); + let error = clone.fails(&["log", "main"]); + assert!(error.contains("main"), "not followed: {error}"); + let fetched = clone.ok(&["fetch", "--target", "main"]); + assert!(fetched.contains("`main`: now followed, at #1"), "{fetched}"); + assert!(clone.ok(&["log", "main"]).starts_with("◆ main #1")); + + let missing = clone.fails(&["clone", url, "elsewhere", "--target", "nope"]); + assert!(missing.contains("no target `nope`"), "{missing}"); + assert!( + !clone.path("elsewhere").exists(), + "a failed clone leaves nothing" + ); +} + +#[test] +fn a_move_that_cannot_fetch_moves_nothing() { + let ada = User::new("ada"); + let origin = Repo::init(&ada); + origin.write("x", "x\n"); + origin.ok(&["describe", "X"]); + let change = origin.full_change(); + let url = origin.root().to_str().unwrap(); + let clone = Repo::clone(&ada, url, &[]); + clone.ok(&["fetch", "--change", &change]); + + // The remote is gone while `edit` needs the change's files from it. + let away = origin.root().with_extension("away"); + fs::rename(origin.root(), &away).unwrap(); + let error = clone.fails(&["edit", &change]); + assert!(error.contains("remote"), "{error}"); + assert!(!clone.path(".origin/moving").exists()); + clone.write("notes", "mine\n"); + let status = clone.ok(&["status"]); + assert!(status.contains("A notes\n"), "{status}"); + fs::rename(&away, origin.root()).unwrap(); + clone.ok(&["edit", &change]); + assert_eq!(clone.read("x"), "x\n"); +} diff --git a/crates/origin-client/src/lib.rs b/crates/origin-client/src/lib.rs index 398f2d9c..68523f5f 100644 --- a/crates/origin-client/src/lib.rs +++ b/crates/origin-client/src/lib.rs @@ -19,27 +19,28 @@ //! in `docs/openapi.json`, for other languages. use origin_shared::api::{ - Annotation, ApiError, EventsQuery, FetchRecords, LandRequest, Landings, LogEntry, LogQuery, - NewTarget, RepoName, RepoSummary, Revisions, StatementsQuery, Stored, TargetInfo, + Annotation, ApiError, EventsQuery, FetchRecords, FetchSnapshot, Inventory, InventoryRequest, + LandRequest, Landings, LogEntry, LogQuery, MORE, NewTarget, PutQuery, RepoName, RepoSummary, + Revisions, StatementsQuery, Stored, TargetInfo, }; use origin_shared::event::Page; use origin_shared::landing::Pending; -use origin_shared::pack::{Pack, PackError}; +use origin_shared::pack::{Pack, PackError, Part}; use origin_shared::{ChangeId, Head}; use reqwest::{Method, RequestBuilder}; use serde::de::DeserializeOwned; /// Defines endpoints as methods. Each sends one request, built from an HTTP /// method, a path relative to its owner's, which can name the arguments -/// like `format!` does, and optionally a body applied by a -/// [`RequestBuilder`] method (`query`, `json` or `body`), then reads the -/// response with a [`Response`] method. +/// like `format!` does, and what [`RequestBuilder`] methods (`query`, +/// `json` or `body`) apply, then reads the response with a [`Response`] +/// method. macro_rules! endpoints { ($( impl $owner:ident {$( $(#[doc = $doc:literal])* fn $name:ident($($arg:ident: $type:ty),*) -> $output:ty = - $method:ident $path:literal $(, $send:ident($body:expr))? => $read:ident; + $method:ident $path:literal $(, $send:ident($body:expr))* => $read:ident; )*} )*) => {$( impl $owner {$( @@ -49,7 +50,7 @@ macro_rules! endpoints { /// /// Fails if the server cannot be reached or refuses the request. pub async fn $name(&self, $($arg: $type),*) -> Result<$output, Error> { - let request = self.request(Method::$method, &format!($path))$(.$send($body))?; + let request = self.request(Method::$method, &format!($path))$(.$send($body))*; Response::send(request).await?.$read().await } )*} @@ -65,16 +66,34 @@ endpoints! { } impl Repo { - /// Fetches objects by id, in the order asked. - fn fetch_records(request: &FetchRecords) -> Pack = - POST "records/fetch", json(request) => pack; + /// Fetches records by id, in the order asked, each followed by the + /// blob it carries if the server has it: a revision's description, + /// a statement's payload. A part that stops short holds those up to + /// the last record it has. + fn fetch_records(request: &FetchRecords) -> Part = + POST "records/fetch", json(request) => part; + /// Fetches the trees and blobs of a snapshot that none of the + /// request's snapshots holds at the same path, parents first. A + /// part that stops short is followed by asking again, skipping the + /// objects it holds. + fn fetch_snapshot(request: &FetchSnapshot) -> Part = + POST "snapshots/fetch", json(request) => part; /// Puts a batch of objects: snapshots, revisions and statements, /// each needing only objects the batch or the repository has. fn put_records(pack: &Pack) -> Stored = PUT "records", body(pack.encode()) => json; + /// Puts a batch of objects like `put_records`, if each change the + /// query names has the heads it says when its revisions are + /// published: a compare-and-swap, refused with a conflict. + fn push_records(pack: &Pack, query: &PutQuery) -> Stored = + PUT "records", query(query), body(pack.encode()) => json; /// The statements about a record, in the order the repository /// received them. fn statements(query: &StatementsQuery) -> Vec = GET "statements", query(query) => json; + /// What the server holds of some changes: their revisions, heads + /// and statements, by id. + fn inventory(request: &InventoryRequest) -> Vec = + POST "changes/inventory", json(request) => json; /// The revisions of a change, and its heads. fn revisions(change: ChangeId) -> Revisions = GET "changes/{change}/revisions" => json; /// Every target, with its tip, authority and requirements. @@ -170,8 +189,10 @@ impl Response { Ok(self.0.json().await?) } - async fn pack(self) -> Result { - Ok(Pack::decode(&self.0.bytes().await?)?) + async fn part(self) -> Result { + let more = self.0.headers().contains_key(MORE); + let pack = Pack::decode(&self.0.bytes().await?)?; + Ok(Part { pack, more }) } } diff --git a/crates/origin-shared/src/api.rs b/crates/origin-shared/src/api.rs index c0b1caf6..85cb2e11 100644 --- a/crates/origin-shared/src/api.rs +++ b/crates/origin-shared/src/api.rs @@ -11,8 +11,8 @@ use crate::landing::{Landing, Proposal}; use crate::policy::{KindPattern, Requirement}; use crate::text::Period; use crate::{ - Acceptance, AcceptanceId, Head, ObjectId, PrincipalName, RecordId, Revision, RevisionId, - Statement, StatementId, TargetName, + Acceptance, AcceptanceId, ChangeId, Head, ObjectId, PrincipalName, RecordId, RepoPath, + Revision, RevisionId, Statement, StatementId, TargetName, TreeId, }; /// The name of a repository on a server: ASCII letters, digits, `.`, `_` @@ -95,13 +95,132 @@ pub struct RepoSummary { pub name: RepoName, } -/// Asks for records, and any other objects, by id. +/// The response header saying a fetch stopped short of what it asked for, +/// to stay within [`Pack::LIMIT`](crate::pack::Pack::LIMIT): see +/// [`Part`](crate::pack::Part). +pub const MORE: &str = "origin-more"; + +/// Asks for records by id: revisions, statements and acceptances. Each +/// comes with the blob it carries, if the repository has it: a revision's +/// description, a statement's payload. Snapshots are fetched with +/// [`FetchSnapshot`] instead. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct FetchRecords { - /// The objects, returned in this order. + /// The records, returned in this order, each followed by its blob. A + /// response that stops short, with the [`MORE`] header, holds those up + /// to the last record it has: ask again for those after it. pub ids: Vec, } +/// Asks for the objects of a snapshot that the caller lacks: its trees and +/// blobs, walked from its root. This is the only way blobs and trees leave +/// a repository, so what each caller may read can be checked by path. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct FetchSnapshot { + /// The snapshot. + pub snapshot: TreeId, + /// Snapshots the caller holds whole. Whatever one of them holds at the + /// same path is skipped, so only what changed since comes. Those the + /// repository doesn't have are ignored. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub have: Vec, + /// Only these files and directories, with everything under them, and + /// the trees on the way to them. Everything by default. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub paths: Vec, + /// Skip this many of the objects, which earlier parts brought: a + /// response that stops short has the [`MORE`] header. + #[serde(default, skip_serializing_if = "is_zero")] + pub skip: u64, +} + +#[expect( + clippy::trivially_copy_pass_by_ref, + reason = "serde passes a reference" +)] +fn is_zero(value: &u64) -> bool { + *value == 0 +} + +/// Asks what a repository holds of some changes. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct InventoryRequest { + /// The changes. + pub changes: Vec, +} + +/// What a repository holds of a change: its records, by id, to compare +/// with another node's. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct Inventory { + /// The change. + pub change: ChangeId, + /// Its revisions, in the order the repository received them: none if it + /// has never heard of the change. + pub revisions: Vec, + /// Those no other revision of the change evolves from. + pub heads: Vec, + /// The statements about its revisions, and about those statements, in + /// no particular order. + pub statements: Vec, +} + +/// Asks for a batch of objects to be put only if the changes named are where +/// the caller saw them: a compare-and-swap, so a push made from a stale view +/// of a change is refused instead of making it diverge. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, IntoParams)] +#[into_params(parameter_in = Query)] +pub struct PutQuery { + /// The heads of changes as the caller saw them, comma-separated, each + /// `=+`, or `=` for a change the repository + /// didn't have. The batch is refused if one's heads differ now. + #[serde(default, skip_serializing_if = "List::is_empty")] + #[param(value_type = Option)] + pub expect: List, +} + +/// The heads of a change as someone saw them, written +/// `=+`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Expected { + /// The change. + pub change: ChangeId, + /// Its heads: none if the repository didn't have it. + pub heads: Vec, +} + +impl fmt::Display for Expected { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}=", self.change)?; + for (i, head) in self.heads.iter().enumerate() { + let separator = if i == 0 { "" } else { "+" }; + write!(f, "{separator}{head}")?; + } + Ok(()) + } +} + +impl FromStr for Expected { + type Err = String; + + fn from_str(input: &str) -> Result { + let invalid = |error: crate::ParseIdError| format!("invalid heads {input:?}: {error}"); + let (change, heads) = input + .split_once('=') + .ok_or_else(|| format!("invalid heads {input:?}: expected =+"))?; + let heads = heads + .split('+') + .filter(|head| !head.is_empty()) + .map(str::parse) + .collect::>() + .map_err(invalid)?; + Ok(Self { + change: change.parse().map_err(invalid)?, + heads, + }) + } +} + /// What putting a batch of records did. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct Stored { @@ -177,6 +296,8 @@ pub struct LogQuery { /// default. #[serde(default)] pub since: u64, + /// Return at most this many acceptances: all by default. + pub limit: Option, } /// One acceptance of a target's log. @@ -225,6 +346,10 @@ pub struct EventsQuery { /// The cursor to read after: the `next` of the previous page. By /// default, the end of the stream, so only new events come. pub cursor: Option, + /// The generation of the op log the cursor is of, as its page said. A + /// cursor of another generation, such as from before the repository + /// was restored, has expired. + pub generation: Option, /// How long to wait for events, such as `30s`. Without it, the answer /// comes at once, possibly empty. #[param(value_type = Option)] @@ -334,6 +459,23 @@ mod tests { } } + #[test] + fn expected_heads_round_trip() { + let change = ChangeId::from_bytes([1; 16]); + let heads = [ + RevisionId::from_bytes([2; 32]), + RevisionId::from_bytes([3; 32]), + ]; + for heads in [&heads[..], &heads[..1], &[]] { + let expected = Expected { + change, + heads: heads.to_vec(), + }; + assert_eq!(expected.to_string().parse::(), Ok(expected)); + } + assert!("nonsense".parse::().is_err()); + } + #[test] fn lists_are_comma_separated() { let list: List = serde_json::from_str("\"ci.*,review\"").unwrap(); diff --git a/crates/origin-shared/src/event.rs b/crates/origin-shared/src/event.rs index 2c587b12..a3f06c51 100644 --- a/crates/origin-shared/src/event.rs +++ b/crates/origin-shared/src/event.rs @@ -254,6 +254,10 @@ pub struct Page { pub events: Vec, /// The cursor after the entries read: pass it to the next read. pub next: u64, + /// The generation of the op log the cursor is of: pass it with the + /// cursor, which means nothing in another. + #[serde(default)] + pub generation: String, } #[cfg(test)] diff --git a/crates/origin-shared/src/landing.rs b/crates/origin-shared/src/landing.rs index 434f02f2..a23a82d0 100644 --- a/crates/origin-shared/src/landing.rs +++ b/crates/origin-shared/src/landing.rs @@ -65,6 +65,18 @@ pub enum Blocked { }, } +impl Landing { + /// What the landing lands. + #[must_use] + pub fn proposal(&self) -> &Proposal { + match self { + Self::Landed { proposal, .. } + | Self::Unchanged(proposal) + | Self::Pending(Pending { proposal, .. }) => proposal, + } + } +} + impl fmt::Display for Proposal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { diff --git a/crates/origin-shared/src/pack.rs b/crates/origin-shared/src/pack.rs index f005f1f0..34e1e881 100644 --- a/crates/origin-shared/src/pack.rs +++ b/crates/origin-shared/src/pack.rs @@ -4,6 +4,10 @@ //! `id:[u8; 32] len:u64 bytes:[u8; len]` with `bytes` the object's encoding. //! Decoding verifies every object against its id, so a pack can come from an //! untrusted peer. +//! +//! Transfers are bounded: a [`Packer`] splits objects into packs of at most +//! [`Pack::LIMIT`] bytes, in order, so a push takes several requests, and a +//! fetch response stops at a [`Part`], with the rest asked for again. use crate::object::{DecodeError, IdMismatch, Reader}; use crate::{EncodedObject, ObjectId}; @@ -18,14 +22,20 @@ pub struct Pack { impl Pack { const HEADER_LEN: usize = ObjectId::LEN + 8; + /// The most bytes a request or a response carries as one pack, encoded. + /// A pack holds one object at least, so an object larger than this + /// travels alone. + pub const LIMIT: usize = 64 << 20; + + /// The bytes `object` takes in an encoded pack. + fn encoded_len(object: &EncodedObject) -> usize { + Self::HEADER_LEN + object.bytes().len() + } + /// Encodes the pack. #[must_use] pub fn encode(&self) -> Vec { - let len = self - .objects - .iter() - .map(|object| Self::HEADER_LEN + object.bytes().len()) - .sum(); + let len = self.objects.iter().map(Self::encoded_len).sum(); let mut out = Vec::with_capacity(len); for object in &self.objects { out.extend_from_slice(object.id().as_bytes()); @@ -53,6 +63,97 @@ impl Pack { } } +/// Splits objects into packs of at most a limit, [`Pack::LIMIT`] by +/// default, in the order they're added. Objects only ever depend on those +/// before them, so each pack can be sent as soon as it's full. +#[derive(Debug)] +pub struct Packer { + limit: usize, + pack: Pack, + /// The encoded size of `pack`. + len: usize, +} + +impl Default for Packer { + fn default() -> Self { + Self::new(Pack::LIMIT) + } +} + +impl Packer { + /// A packer filling packs up to `limit` bytes. + #[must_use] + pub fn new(limit: usize) -> Self { + Self { + limit, + pack: Pack::default(), + len: 0, + } + } + + /// Whether the pack being filled has room for `objects`: an empty one + /// has room for anything. + #[must_use] + pub fn fits<'o>(&self, objects: impl IntoIterator) -> bool { + let len: usize = objects.into_iter().map(Pack::encoded_len).sum(); + self.pack.objects.is_empty() || self.len + len <= self.limit + } + + /// Adds `object` to the pack being filled. If it has no room for it, + /// returns that pack, full, and starts the next one with the object. + #[must_use = "a full pack is returned, not kept"] + pub fn add(&mut self, object: EncodedObject) -> Option { + self.add_together(vec![object]) + } + + /// Adds `objects` to the pack being filled, which they all go in. If it + /// has no room for them, returns that pack, full, and starts the next + /// one with them. + #[must_use = "a full pack is returned, not kept"] + pub fn add_together(&mut self, objects: Vec) -> Option { + let full = (!self.fits(&objects)).then(|| { + self.len = 0; + std::mem::take(&mut self.pack) + }); + self.extend(objects); + full + } + + /// Adds `objects` to the pack being filled, room or not: those + /// [`Packer::fits`] said it has room for. + pub fn extend(&mut self, objects: impl IntoIterator) { + for object in objects { + self.len += Pack::encoded_len(&object); + self.pack.objects.push(object); + } + } + + /// Whether no object was added since the last full pack. + #[must_use] + pub fn is_empty(&self) -> bool { + self.pack.objects.is_empty() + } + + /// The pack being filled. + #[must_use] + pub fn finish(self) -> Pack { + self.pack + } +} + +/// A fetch's answer, which may stop short of everything asked for, to stay +/// within [`Pack::LIMIT`]: then `more` is set, and asking again for the +/// rest brings the next part (see +/// [`FetchRecords`](crate::api::FetchRecords) and +/// [`FetchSnapshot`](crate::api::FetchSnapshot)). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Part { + /// The objects of this part. + pub pack: Pack, + /// Whether it stopped short. + pub more: bool, +} + /// Bytes that are not a valid pack. #[derive(Debug, thiserror::Error)] pub enum PackError { @@ -78,6 +179,25 @@ mod tests { assert_eq!(Pack::decode(&[]).unwrap(), Pack::default()); } + #[test] + fn packers_split_in_order_within_the_limit() { + let large = [b'x'; 100]; + let objects: Vec<_> = [b"a".as_slice(), b"bb", b"c", &large] + .map(EncodedObject::blob) + .to_vec(); + let limit = 2 * Pack::encoded_len(&objects[0]) + 1; + let mut packer = Packer::new(limit); + let mut packs: Vec = objects + .iter() + .filter_map(|object| packer.add(object.clone())) + .collect(); + packs.push(packer.finish()); + let split: Vec = packs.iter().map(|pack| pack.objects.len()).collect(); + assert_eq!(split, [2, 1, 1], "one larger than the limit travels alone"); + let joined: Vec<_> = packs.into_iter().flat_map(|pack| pack.objects).collect(); + assert_eq!(joined, objects, "in order"); + } + #[test] fn rejects_truncated_and_corrupt_packs() { let mut bytes = Pack { diff --git a/docs/openapi.json b/docs/openapi.json index 3e2e4de0..1e9db4e8 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -48,6 +48,58 @@ } } }, + "/api/v1/repos/{repo}/changes/inventory": { + "post": { + "summary": "Lists what the repository holds of some changes: their revisions,\nheads and statements, by id.", + "operationId": "inventory", + "parameters": [ + { + "name": "repo", + "in": "path", + "description": "The repository", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InventoryRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "An inventory for each change, in the order asked: empty for a change the repository never heard of", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Inventory" + } + } + } + } + }, + "404": { + "description": "The repository does not exist", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + } + } + }, "/api/v1/repos/{repo}/changes/{change}/revisions": { "get": { "summary": "Lists the revisions of a change, and its heads.", @@ -121,6 +173,15 @@ "minimum": 0 } }, + { + "name": "generation", + "in": "query", + "description": "The generation of the op log the cursor is of, as its page said. A\ncursor of another generation, such as from before the repository\nwas restored, has expired.", + "required": false, + "schema": { + "type": "string" + } + }, { "name": "wait", "in": "query", @@ -338,6 +399,17 @@ "format": "int64", "minimum": 0 } + }, + { + "name": "limit", + "in": "query", + "description": "Return at most this many acceptances: all by default.", + "required": false, + "schema": { + "type": "integer", + "format": "int64", + "minimum": 0 + } } ], "responses": { @@ -369,7 +441,7 @@ }, "/api/v1/repos/{repo}/records": { "put": { - "summary": "Puts a batch of objects, as a pack: snapshots, revisions and\nstatements, each needing only objects the batch or the repository has,\nof the kinds it names.", + "summary": "Puts a batch of objects, as a pack: snapshots, revisions and\nstatements, each needing only objects the batch or the repository has,\nof the kinds it names. With `expect`, only if the changes it names have\nthe heads it says when the records are published.", "operationId": "put_records", "parameters": [ { @@ -380,6 +452,15 @@ "schema": { "type": "string" } + }, + { + "name": "expect", + "in": "query", + "description": "The heads of changes as the caller saw them, comma-separated, each\n`=+`, or `=` for a change the repository\ndidn't have. The batch is refused if one's heads differ now.", + "required": false, + "schema": { + "type": "string" + } } ], "requestBody": { @@ -423,13 +504,23 @@ } } } + }, + "409": { + "description": "A change of `expect` has other heads now", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } } } } }, "/api/v1/repos/{repo}/records/fetch": { "post": { - "summary": "Fetches objects by id, as a pack.", + "summary": "Fetches records by id, each with the blob it carries, as a pack.", "operationId": "fetch_records", "parameters": [ { @@ -454,7 +545,82 @@ }, "responses": { "200": { - "description": "The objects, in the order asked, as a pack: each a 32-byte id, a little-endian u64 length and the object's encoding", + "description": "The records, in the order asked, each followed by its description or payload if the repository has it, as a pack: each object a 32-byte id, a little-endian u64 length and the object's encoding. At most 64 MiB, but for a record alone", + "headers": { + "origin-more": { + "schema": { + "type": "boolean" + }, + "description": "Present if the pack stopped short: ask again for the records after the last it holds" + } + }, + "content": { + "application/octet-stream": { + "schema": { + "$ref": "#/components/schemas/Binary" + } + } + } + }, + "400": { + "description": "An object is a blob or a tree, which only leave by walking a snapshot", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "The repository or a record does not exist", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + } + } + }, + "/api/v1/repos/{repo}/snapshots/fetch": { + "post": { + "summary": "Fetches the objects of a snapshot the caller lacks, as a pack: the only\nway trees and blobs leave a repository.", + "operationId": "fetch_snapshot", + "parameters": [ + { + "name": "repo", + "in": "path", + "description": "The repository", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FetchSnapshot" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "The trees and blobs, parents before children, as a pack, of at most 64 MiB, but for an object alone", + "headers": { + "origin-more": { + "schema": { + "type": "boolean" + }, + "description": "Present if the pack stopped short: ask again, skipping the objects it holds" + } + }, "content": { "application/octet-stream": { "schema": { @@ -464,7 +630,7 @@ } }, "404": { - "description": "The repository or an object does not exist", + "description": "The repository or the snapshot does not exist", "content": { "application/json": { "schema": { @@ -1052,7 +1218,7 @@ }, "FetchRecords": { "type": "object", - "description": "Asks for records, and any other objects, by id.", + "description": "Asks for records by id: revisions, statements and acceptances. Each\ncomes with the blob it carries, if the repository has it: a revision's\ndescription, a statement's payload. Snapshots are fetched with\n[`FetchSnapshot`] instead.", "required": [ "ids" ], @@ -1062,7 +1228,40 @@ "items": { "$ref": "#/components/schemas/ObjectId" }, - "description": "The objects, returned in this order." + "description": "The records, returned in this order, each followed by its blob. A\nresponse that stops short, with the [`MORE`] header, holds those up\nto the last record it has: ask again for those after it." + } + } + }, + "FetchSnapshot": { + "type": "object", + "description": "Asks for the objects of a snapshot that the caller lacks: its trees and\nblobs, walked from its root. This is the only way blobs and trees leave\na repository, so what each caller may read can be checked by path.", + "required": [ + "snapshot" + ], + "properties": { + "have": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TreeId" + }, + "description": "Snapshots the caller holds whole. Whatever one of them holds at the\nsame path is skipped, so only what changed since comes. Those the\nrepository doesn't have are ignored." + }, + "paths": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RepoPath" + }, + "description": "Only these files and directories, with everything under them, and\nthe trees on the way to them. Everything by default." + }, + "skip": { + "type": "integer", + "format": "int64", + "description": "Skip this many of the objects, which earlier parts brought: a\nresponse that stops short has the [`MORE`] header.", + "minimum": 0 + }, + "snapshot": { + "$ref": "#/components/schemas/TreeId", + "description": "The snapshot." } } }, @@ -1117,6 +1316,59 @@ } } }, + "Inventory": { + "type": "object", + "description": "What a repository holds of a change: its records, by id, to compare\nwith another node's.", + "required": [ + "change", + "revisions", + "heads", + "statements" + ], + "properties": { + "change": { + "$ref": "#/components/schemas/ChangeId", + "description": "The change." + }, + "heads": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RevisionId" + }, + "description": "Those no other revision of the change evolves from." + }, + "revisions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RevisionId" + }, + "description": "Its revisions, in the order the repository received them: none if it\nhas never heard of the change." + }, + "statements": { + "type": "array", + "items": { + "$ref": "#/components/schemas/StatementId" + }, + "description": "The statements about its revisions, and about those statements, in\nno particular order." + } + } + }, + "InventoryRequest": { + "type": "object", + "description": "Asks what a repository holds of some changes.", + "required": [ + "changes" + ], + "properties": { + "changes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ChangeId" + }, + "description": "The changes." + } + } + }, "Kind": { "type": "string", "examples": [ @@ -1288,6 +1540,10 @@ }, "description": "The events, oldest first." }, + "generation": { + "type": "string", + "description": "The generation of the op log the cursor is of: pass it with the\ncursor, which means nothing in another." + }, "next": { "type": "integer", "format": "int64", diff --git a/tools/xtask/src/main.rs b/tools/xtask/src/main.rs index 99c27f8f..3649b29f 100644 --- a/tools/xtask/src/main.rs +++ b/tools/xtask/src/main.rs @@ -1,16 +1,17 @@ //! Development tasks, run with `cargo xtask `: //! //! - `bench-git [--files N] [--tree PATH]`: times origin against git on the -//! same working tree with hyperfine, and writes `docs/benchmarks.md`. The -//! tree is a synthetic fixture of N files, or a copy of PATH. +//! same working tree with hyperfine, locally and syncing with a server, +//! and writes `docs/benchmarks.md`. The tree is a synthetic fixture of N +//! files, or a copy of PATH. //! - `edit DIR COUNT`: appends a line to COUNT files spread across DIR. The //! benchmarks use it to change files between runs. use std::fmt::Write as _; use std::fs::{self, OpenOptions}; -use std::io::{self, Write as _}; +use std::io::{self, BufRead, BufReader, Write as _}; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::process::{Child, Command, Stdio}; use std::time::{SystemTime, UNIX_EPOCH}; use std::{env, iter}; @@ -73,12 +74,6 @@ struct BenchGit { } impl BenchGit { - /// `path` as one `sh` word, whatever it holds: single-quoted, with each - /// `'` closing the quote, escaped, and reopening it. - fn quote(path: &Path) -> String { - format!("'{}'", path.display().to_string().replace('\'', "'\\''")) - } - fn parse(mut args: impl Iterator) -> Result { let mut task = Self { files: FixtureTree::LARGE.files, @@ -113,8 +108,6 @@ impl BenchGit { .current_dir(&workspace))?; // Cargo resolves a relative target directory from where it runs. let target = workspace.join(env::var_os("CARGO_TARGET_DIR").unwrap_or("target".into())); - let origin = target.join("release/origin").canonicalize()?; - let xtask = env::current_exe()?; let base = target.join("bench-git"); if base.exists() { @@ -123,86 +116,24 @@ impl BenchGit { let config = base.join("config"); fs::create_dir_all(&config)?; fs::write(config.join("config.toml"), "user = \"bench\"\n")?; - let (origin_dir, git_dir) = (base.join("origin"), base.join("git")); - for dir in [&origin_dir, &git_dir] { - self.populate(dir)?; + for dir in ["origin", "git"] { + self.populate(&base.join(dir))?; } - let (origin_dir, git_dir) = (origin_dir.canonicalize()?, git_dir.canonicalize()?); - let shell = |dir: &Path, command: &str| format!("cd {} && {command}", Self::quote(dir)); - let origin_cmd = - |command: &str| shell(&origin_dir, &format!("{} {command}", Self::quote(&origin))); - let git_cmd = |command: &str| shell(&git_dir, command); - let edit_cmd = - |dir: &Path| format!("{} edit {} 100", Self::quote(&xtask), Self::quote(dir)); - - let mut report = self.report_header()?; - let mut scenario = |title: &str, - runs: &[&str], - origin: (Option, String), - git: (Option, String)| { - let markdown = base.join("scenario.md"); - let mut hyperfine = Command::new("hyperfine"); - // The commands are shell syntax, which hyperfine 2 makes explicit. - hyperfine - .args(runs) - .arg("--shell=sh") - .arg("--export-markdown") - .arg(&markdown); - for (name, (prepare, command)) in [("origin", origin), ("git", git)] { - if let Some(prepare) = prepare { - hyperfine.arg("--prepare").arg(prepare); - } - hyperfine.arg("--command-name").arg(name).arg(command); - } - run(hyperfine - .envs(Self::environment()) - .env("ORIGIN_CONFIG_DIR", &config))?; - writeln!(report, "\n## {title}\n\n{}", fs::read_to_string(&markdown)?)?; - anyhow::Ok(()) + let mut bench = Bench { + origin: target.join("release/origin").canonicalize()?, + xtask: env::current_exe()?, + origin_dir: base.join("origin").canonicalize()?, + git_dir: base.join("git").canonicalize()?, + config, + base, + report: self.report_header()?, }; - - // Origin has no commit: `new` records the working copy into the - // current change and starts the next one, the closest operation. - scenario( - "First commit of the whole tree (`origin new`)", - &["--runs", "5"], - ( - Some(shell( - &origin_dir, - &format!("rm -rf .origin && {} init", Self::quote(&origin)), - )), - origin_cmd("new bench"), - ), - ( - Some(git_cmd("rm -rf .git && git init -q")), - git_cmd("git add -A && git commit -qm bench"), - ), - )?; - scenario( - "Status, nothing changed", - &["--warmup", "3"], - (None, origin_cmd("status")), - (None, git_cmd("git status --porcelain")), - )?; - scenario( - "Status, 100 files edited", - &["--warmup", "1"], - (Some(edit_cmd(&origin_dir)), origin_cmd("status")), - (Some(edit_cmd(&git_dir)), git_cmd("git status --porcelain")), - )?; - scenario( - "Commit, 100 files edited (`origin new`)", - &["--warmup", "1"], - (Some(edit_cmd(&origin_dir)), origin_cmd("new bench")), - ( - Some(edit_cmd(&git_dir)), - git_cmd("git add -A && git commit -qm bench"), - ), - )?; + bench.local()?; + bench.sync(self)?; let path = workspace.join("docs/benchmarks.md"); fs::create_dir_all(path.parent().context("docs has a parent")?)?; - fs::write(&path, report)?; + fs::write(&path, bench.report)?; println!("Wrote {}", path.canonicalize()?.display()); Ok(()) } @@ -212,13 +143,18 @@ impl BenchGit { fs::create_dir_all(dir)?; match &self.tree { Some(tree) => { - run(Command::new("cp").arg("-a").arg(tree.join(".")).arg(dir))?; - for metadata in [".git", ".origin"] { - let path = dir.join(metadata); - if path.exists() { - fs::remove_dir_all(path)?; + // All but the tree's own metadata: `dir` may be a clone, + // whose own must stay. + let mut entries = Vec::new(); + for entry in fs::read_dir(tree)? { + let path = entry?.path(); + if !path.ends_with(".git") && !path.ends_with(".origin") { + entries.push(path); } } + if !entries.is_empty() { + run(Command::new("cp").arg("-a").args(entries).arg(dir))?; + } } None => FixtureTree { files: self.files, @@ -270,6 +206,283 @@ impl BenchGit { } } +/// One side of a scenario: a command preparing each run, if any, and the +/// timed command. +type Side = (Option, String); + +/// A benchmark run: the two working copies under test, and the report. +struct Bench { + origin: PathBuf, + xtask: PathBuf, + base: PathBuf, + config: PathBuf, + origin_dir: PathBuf, + git_dir: PathBuf, + report: String, +} + +impl Bench { + /// `path` as one `sh` word, whatever it holds: single-quoted, with each + /// `'` closing the quote, escaped, and reopening it. + fn quote(path: &Path) -> String { + format!("'{}'", path.display().to_string().replace('\'', "'\\''")) + } + + fn shell(dir: &Path, command: &str) -> String { + format!("cd {} && {command}", Self::quote(dir)) + } + + /// An origin command in `dir`. + fn origin_in(&self, dir: &Path, command: &str) -> String { + Self::shell(dir, &format!("{} {command}", Self::quote(&self.origin))) + } + + /// An origin command in origin's working copy. + fn origin(&self, command: &str) -> String { + self.origin_in(&self.origin_dir, command) + } + + /// A git command in git's working copy. + fn git(&self, command: &str) -> String { + Self::shell(&self.git_dir, command) + } + + /// Edits 100 files of a working copy. + fn edit(&self, dir: &Path) -> String { + format!("{} edit {} 100", Self::quote(&self.xtask), Self::quote(dir)) + } + + /// Runs origin with `args` in `dir`, outside of hyperfine. + fn run_origin(&self, dir: &Path, args: &[&str]) -> Result<()> { + run(Command::new(&self.origin) + .args(args) + .current_dir(dir) + .env("ORIGIN_CONFIG_DIR", &self.config) + .stdout(Stdio::null())) + } + + /// Runs git with `args` in `dir`, outside of hyperfine. + fn run_git(dir: &Path, args: &[&str]) -> Result<()> { + run(Command::new("git") + .args(args) + .current_dir(dir) + .envs(BenchGit::environment()) + .stdout(Stdio::null())) + } + + /// Times both sides with hyperfine and appends the results to the report. + fn scenario(&mut self, title: &str, runs: &[&str], origin: Side, git: Side) -> Result<()> { + let markdown = self.base.join("scenario.md"); + let mut hyperfine = Command::new("hyperfine"); + // The commands are shell syntax, which hyperfine 2 makes explicit. + hyperfine + .args(runs) + .arg("--shell=sh") + .arg("--export-markdown") + .arg(&markdown); + for (name, (prepare, command)) in [("origin", origin), ("git", git)] { + if let Some(prepare) = prepare { + hyperfine.arg("--prepare").arg(prepare); + } + hyperfine.arg("--command-name").arg(name).arg(command); + } + run(hyperfine + .envs(BenchGit::environment()) + .env("ORIGIN_CONFIG_DIR", &self.config))?; + writeln!( + self.report, + "\n## {title}\n\n{}", + fs::read_to_string(&markdown)? + )?; + Ok(()) + } + + /// Recording and inspecting the working copy. Origin has no commit: + /// `new` records the working copy into the current change and starts the + /// next one, the closest operation. + fn local(&mut self) -> Result<()> { + let init = Self::shell( + &self.origin_dir, + &format!("rm -rf .origin && {} init", Self::quote(&self.origin)), + ); + self.scenario( + "First commit of the whole tree (`origin new`)", + &["--runs", "5"], + (Some(init), self.origin("new bench")), + ( + Some(self.git("rm -rf .git && git init -q -b main")), + self.git("git add -A && git commit -qm bench"), + ), + )?; + self.scenario( + "Status, nothing changed", + &["--warmup", "3"], + (None, self.origin("status")), + (None, self.git("git status --porcelain")), + )?; + self.scenario( + "Status, 100 files edited", + &["--warmup", "1"], + (Some(self.edit(&self.origin_dir)), self.origin("status")), + ( + Some(self.edit(&self.git_dir)), + self.git("git status --porcelain"), + ), + )?; + self.scenario( + "Commit, 100 files edited (`origin new`)", + &["--warmup", "1"], + (Some(self.edit(&self.origin_dir)), self.origin("new bench")), + ( + Some(self.edit(&self.git_dir)), + self.git("git add -A && git commit -qm bench"), + ), + ) + } + + /// Cloning, pushing and pulling. Origin serves over HTTP on localhost; + /// git uses its pack protocol on a local bare repository, which spares + /// it the network stack. Origin's side starts from a clone with the + /// tree landed, and git's from its working copy, pushed. + fn sync(&mut self, task: &BenchGit) -> Result<()> { + let server = Server::start(&self.origin, &self.base, &self.config)?; + let (seed, clone) = ( + self.base.join("origin-seed"), + self.base.join("origin-clone"), + ); + self.run_origin(&self.base, &["clone", &server.url, &seed.to_string_lossy()])?; + task.populate(&seed)?; + self.run_origin(&seed, &["land"])?; + let (remote, git_clone) = (self.base.join("remote.git"), self.base.join("git-clone")); + let remote_path = remote.to_string_lossy(); + Self::run_git( + &self.base, + &["init", "-q", "--bare", "-b", "main", &remote_path], + )?; + Self::run_git(&self.git_dir, &["push", "-q", &remote_path, "HEAD:main"])?; + let push = format!("git push -q {} HEAD:main", Self::quote(&remote)); + + self.scenario( + "Clone", + &["--runs", "5"], + ( + Some(format!("rm -rf {}", Self::quote(&clone))), + format!( + "{} clone {} {}", + Self::quote(&self.origin), + server.url, + Self::quote(&clone) + ), + ), + ( + Some(format!("rm -rf {}", Self::quote(&git_clone))), + format!( + "git clone -q --no-local {} {}", + Self::quote(&remote), + Self::quote(&git_clone) + ), + ), + )?; + // `describe` records the edits, as `commit` does for git. The push + // still snapshots the working copy, as every origin command does. + self.scenario( + "Push, 100 files edited", + &["--warmup", "1"], + ( + Some(format!( + "{} && {}", + self.edit(&seed), + self.origin_in(&seed, "describe bench") + )), + self.origin_in(&seed, "push"), + ), + ( + Some(format!( + "{} && {}", + self.edit(&self.git_dir), + self.git("git add -A && git commit -qm bench") + )), + self.git(&push), + ), + )?; + self.scenario( + "Pull, 100 files landed", + &["--warmup", "1"], + ( + Some(format!( + "{} && {}", + self.edit(&seed), + self.origin_in(&seed, "land") + )), + self.origin_in(&clone, "pull"), + ), + ( + Some(format!( + "{} && {}", + self.edit(&self.git_dir), + self.git(&format!("git add -A && git commit -qm bench && {push}")) + )), + Self::shell(&git_clone, "git pull -q"), + ), + ) + } +} + +/// An `origin serve` hosting one empty repository, stopped when dropped. +struct Server { + child: Child, + url: String, +} + +impl Server { + fn start(origin: &Path, base: &Path, config: &Path) -> Result { + let dir = base.join("server"); + fs::create_dir_all(&dir)?; + fs::write( + dir.join("origin-server.toml"), + "listen = \"127.0.0.1:0\"\nnode = \"hosted\"\n[repos]\ndemo = \"demo\"\n", + )?; + run(Command::new(origin) + .args(["repo", "create", "demo"]) + .current_dir(&dir) + .stdout(Stdio::null()))?; + // Guarded from the start, so a server that fails to start is killed. + let mut server = Self { + child: Command::new(origin) + .arg("serve") + .current_dir(&dir) + .env("ORIGIN_CONFIG_DIR", config) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?, + url: String::new(), + }; + let mut line = String::new(); + BufReader::new( + server + .child + .stdout + .take() + .context("the server has no stdout")?, + ) + .read_line(&mut line)?; + let base = line + .trim() + .strip_prefix("Listening on ") + .with_context(|| format!("unexpected server output: {line}"))?; + server.url = format!("{base}/demo"); + Ok(server) + } +} + +impl Drop for Server { + fn drop(&mut self) { + // Best effort: the benchmark is over either way. + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + fn run(command: &mut Command) -> Result<()> { let status = match command.status() { Err(error) if error.kind() == io::ErrorKind::NotFound => {