Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions .github/workflows/worker-otlp.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
name: Worker OTLP validation

on:
push:
branches: [main, codex/worker-otlp-tracing]
pull_request:
workflow_dispatch:
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

permissions:
contents: read

concurrency:
group: worker-otlp-${{ github.ref }}
cancel-in-progress: true

env:
ELECTRON_SKIP_BINARY_DOWNLOAD: "1"

jobs:
tracing:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with:
node-version-file: .nvmrc
cache: npm
- run: npm ci --no-audit --no-fund
- run: npm run check
- run: npm test -w @space/observability
- run: npm test -w @space/worker
- run: npm test -w @space/api -- otlp trace-correlation dot-goals memory-consolidation-coordinator memory-mutation-coordinator

repository:
# The main CI workflow already runs the complete public suite on every PR.
# Keep this independent full-source check available for manual diagnosis.
if: github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 75
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with:
node-version-file: .nvmrc
cache: npm
- run: npm ci --no-audit --no-fund
- name: Install ripgrep for real file-search tests
run: sudo apt-get update -qq && sudo apt-get install -y -qq ripgrep
- run: npm run check
- run: npm test
- run: npm run build
if: ${{ !cancelled() }}
1 change: 1 addition & 0 deletions apps/api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
"@space/db": "0.2.5",
"@space/gateway": "0.2.5",
"@space/memory-graph": "0.2.5",
"@space/observability": "0.2.5",
"@space/opencode-control": "0.2.5",
"@space/runtime": "0.2.5",
"@space/streaming": "0.2.5",
Expand Down
35 changes: 22 additions & 13 deletions apps/api/src/dot-goals.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createHash } from "node:crypto";
import { z } from "zod";
import { Client, Connection, WorkflowExecutionAlreadyStartedError, WorkflowNotFoundError } from "@temporalio/client";
import { WorkflowExecutionAlreadyStartedError, WorkflowNotFoundError } from "@temporalio/client";
import type { FastifyInstance, FastifyRequest, RouteShorthandOptions } from "fastify";
import {
createDotGoalInputSchema,
Expand All @@ -16,6 +16,8 @@ import {
} from "@space/contracts";
import { SpaceConflictError, SpaceNotFoundError, type SpaceStore } from "@space/runtime";
import type { DotGoalsRepository, DotsRepository } from "@space/db";
import { connectTemporalClient, type TemporalClientFactory, type TemporalWorkflowClient } from "./temporal-client.js";
import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js";

/**
* Space Dots Stage 4: Goals.
Expand All @@ -26,23 +28,24 @@ import type { DotGoalsRepository, DotsRepository } from "@space/db";
* blocking question, cancellation, three consecutive failures or the budget.
*/
export interface DotGoalWorkflowCoordinator {
start(goalId: string): Promise<void>;
start(goalId: string, traceId?: string): Promise<void>;
wake(goalId: string): Promise<void>;
/** Starts the Goal's workflow when none is running; true when it had to start one. */
ensure?(goalId: string): Promise<boolean>;
ensure?(goalId: string, traceId?: string): Promise<boolean>;
}
export class TemporalDotGoalWorkflowCoordinator implements DotGoalWorkflowCoordinator {
constructor(private readonly options: { address: string; namespace: string; taskQueue: string }) {}
private async client<T>(action: (client: Client) => Promise<T>): Promise<T> {
const connection = await Connection.connect({ address: this.options.address, connectTimeout: "5s" });
try { return await action(new Client({ connection, namespace: this.options.namespace })); }
finally { await connection.close(); }
constructor(private readonly options: { address: string; namespace: string; taskQueue: string; clientFactory?: TemporalClientFactory }) {}
private async client<T>(action: (client: TemporalWorkflowClient) => Promise<T>): Promise<T> {
const lease = await (this.options.clientFactory ?? connectTemporalClient)(this.options);
try { return await action(lease.client); }
finally { await lease.close(); }
}
start(goalId: string) {
start(goalId: string, traceId?: string) {
return this.client(async (client) => {
try {
await client.workflow.start("dotGoalWorkflow", { workflowId: dotGoalWorkflowId(goalId), taskQueue: this.options.taskQueue,
args: [{ goalId }], workflowIdReusePolicy: "REJECT_DUPLICATE" });
args: [{ goalId }], workflowIdReusePolicy: "REJECT_DUPLICATE",
memo: workflowCorrelationMemo({ traceId, traceparent: traceparentForTraceId(traceId) }) });
} catch (error) { if (!(error instanceof WorkflowExecutionAlreadyStartedError)) throw error; }
});
}
Expand All @@ -52,11 +55,12 @@ export class TemporalDotGoalWorkflowCoordinator implements DotGoalWorkflowCoordi
catch (error) { if (!(error instanceof WorkflowNotFoundError)) throw error; }
});
}
ensure(goalId: string) {
ensure(goalId: string, traceId?: string) {
return this.client(async (client) => {
try {
await client.workflow.start("dotGoalWorkflow", { workflowId: dotGoalWorkflowId(goalId), taskQueue: this.options.taskQueue,
args: [{ goalId }], workflowIdReusePolicy: "ALLOW_DUPLICATE" });
args: [{ goalId }], workflowIdReusePolicy: "ALLOW_DUPLICATE",
memo: workflowCorrelationMemo({ traceId, traceparent: traceparentForTraceId(traceId) }) });
return true;
} catch (error) {
if (error instanceof WorkflowExecutionAlreadyStartedError) return false;
Expand Down Expand Up @@ -97,6 +101,7 @@ function requestKeyFor(goalId: string, step: number): string {
return `goal-${createHash("sha256").update(`${goalId}:${step}`).digest("hex").slice(0, 40)}`;
}

/** Bind persisted goal operations to owned conversations and their workflow coordinator. */
export function createDotGoalService(deps: DotGoalServiceDeps) {
async function activeSession(goal: DotGoal) {
const session = await deps.store.getSpaceAgentSession(goal.sessionId);
Expand All @@ -106,6 +111,7 @@ export function createDotGoalService(deps: DotGoalServiceDeps) {
return (await deps.goals.transition(goal.goalId, ["ACTIVE", "PAUSED"], status, reason)) ?? goal;
}

/** Create an owned goal and start its workflow under the initiating request's trace when present. */
async function create(owner: string, dotId: string, raw: unknown, request: FastifyRequest | null): Promise<DotGoal> {
const input = createDotGoalInputSchema.parse(raw);
if (!deps.workflow) throw new DotGoalError(503, "DOT_GOALS_UNAVAILABLE", "Goals need the Space worker runtime.");
Expand All @@ -125,7 +131,10 @@ export function createDotGoalService(deps: DotGoalServiceDeps) {
if ((error as { code?: string }).code === "DOT_GOAL_OPEN") throw new DotGoalError(409, "DOT_GOAL_OPEN", "This conversation already has an open Goal.");
throw error;
}
try { await deps.workflow.start(goal.goalId); }
try {
if (request) await deps.workflow.start(goal.goalId, request.requestIdForSpace);
else await deps.workflow.start(goal.goalId);
}
catch {
goal = await finish(goal, "FAILED", "The Goal runtime could not start. Try again.");
throw new DotGoalError(503, "DOT_GOAL_RUNTIME", "The Goal runtime could not start. Try again.");
Expand Down
9 changes: 8 additions & 1 deletion apps/api/src/memory-consolidation-coordinator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
type MemoryConsolidationWorkflowInput
} from "@space/contracts";
import { SpaceFeatureDisabledError } from "@space/runtime";
import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js";

export { MEMORY_CONSOLIDATION_TASK_QUEUE, MEMORY_CONSOLIDATION_WORKFLOW_TYPE };

Expand All @@ -21,6 +22,7 @@ interface WorkflowStartOptions {
workflowId: string;
workflowIdReusePolicy: "REJECT_DUPLICATE";
workflowIdConflictPolicy: "USE_EXISTING";
memo: Record<string, string>;
}

interface WorkflowStartHandle {
Expand All @@ -37,6 +39,10 @@ function isAlreadyStarted(error: unknown): boolean {
return error instanceof Error && error.name === "WorkflowExecutionAlreadyStartedError";
}

/**
* Validate and idempotently start consolidation with the request's recorded
* HTTP span in its correlation memo; report duplicate starts without retrying.
*/
export async function scheduleMemoryConsolidation(
rawInput: MemoryConsolidationWorkflowInput,
workflowId: string,
Expand All @@ -49,7 +55,8 @@ export async function scheduleMemoryConsolidation(
taskQueue: MEMORY_CONSOLIDATION_TASK_QUEUE,
workflowId,
workflowIdReusePolicy: "REJECT_DUPLICATE",
workflowIdConflictPolicy: "USE_EXISTING"
workflowIdConflictPolicy: "USE_EXISTING",
memo: workflowCorrelationMemo({ traceId: input.traceId, traceparent: traceparentForTraceId(input.traceId) })
});
return {
status: "SCHEDULED",
Expand Down
9 changes: 8 additions & 1 deletion apps/api/src/memory-mutation-coordinator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
type MemoryMutationWorkflowInput
} from "@space/contracts";
import { SpaceFeatureDisabledError } from "@space/runtime";
import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js";

export { MEMORY_MUTATION_TASK_QUEUE, MEMORY_MUTATION_WORKFLOW_TYPE, buildMemoryMutationWorkflowId };

Expand All @@ -22,6 +23,7 @@ interface WorkflowStartOptions {
workflowId: string;
workflowIdReusePolicy: "REJECT_DUPLICATE";
workflowIdConflictPolicy: "USE_EXISTING";
memo: Record<string, string>;
}

interface WorkflowStartHandle {
Expand All @@ -38,6 +40,10 @@ function isAlreadyStarted(error: unknown): boolean {
return error instanceof Error && error.name === "WorkflowExecutionAlreadyStartedError";
}

/**
* Validate and idempotently start a change-set mutation, preserving the request's
* recorded HTTP parent in memo. Existing workflow IDs return ALREADY_SCHEDULED.
*/
export async function scheduleMemoryMutation(
rawInput: MemoryMutationWorkflowInput,
startWorkflow: MemoryMutationWorkflowStarter,
Expand All @@ -51,7 +57,8 @@ export async function scheduleMemoryMutation(
taskQueue: MEMORY_MUTATION_TASK_QUEUE,
workflowId,
workflowIdReusePolicy: "REJECT_DUPLICATE",
workflowIdConflictPolicy: "USE_EXISTING"
workflowIdConflictPolicy: "USE_EXISTING",
memo: workflowCorrelationMemo({ traceId: input.traceId, traceparent: traceparentForTraceId(input.traceId) })
});
return { status: "SCHEDULED", workflowId: handle.workflowId, runId: handle.firstExecutionRunId ?? null };
} catch (error) {
Expand Down
2 changes: 2 additions & 0 deletions apps/api/src/otlp-span-exporter.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
// Keep the original API import path stable while API and workers share one transport.
export * from "@space/observability";
7 changes: 6 additions & 1 deletion apps/api/src/room-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,12 @@ export class TemporalRoomAgentWorkflowCoordinator implements RoomAgentWorkflowCo
taskQueue: this.options.taskQueue,
args: [{ roomId: item.turn.roomId, pending: [], processedCount: 0 }],
signal: ROOM_AGENT_ENQUEUE_SIGNAL,
signalArgs: [item]
signalArgs: [item],
memo: workflowCorrelationMemo({
traceId: item.turn.traceId,
traceparent: item.turn.traceparent,
missionId: item.missionId
})
});
recordWorkflowStart(ROOM_AGENT_SUPERVISOR_WORKFLOW_TYPE, "SIGNALED");
return { workflowId, runId: handle.signaledRunId };
Expand Down
4 changes: 3 additions & 1 deletion apps/api/src/trace-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,9 @@ export const requestTraceContexts = new TraceContextRegistry();

/** traceparent for work started on behalf of this Space request id, if known. */
export function traceparentForTraceId(traceId: string | null | undefined, registry = requestTraceContexts): string | null {
return childTraceparent(registry.lookup(traceId)?.traceparent ?? null);
// Propagate the recorded HTTP SERVER span itself. Creating a child id here
// would point the workflow at a span that was never emitted.
return registry.lookup(traceId)?.traceparent ?? null;
}

/** Adds the request's traceparent to a turn input unless it already has a valid one. */
Expand Down
6 changes: 6 additions & 0 deletions apps/api/tests/dot-goals.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ function fixture(overrides: { runStatus?: string; pendingApproval?: boolean; sen
const input = { sessionId: "agent_session:1", objective: "Ship the report", milestones: ["Draft", "Review"], criteria: ["Report published"], budget: { maxSteps: 2 } };

describe("Dot Goals", () => {
it("passes the original API request id when starting a Goal workflow", async () => {
const { service, deps } = fixture();
const goal = await service.create("user:1", dotId, input, { requestIdForSpace: "req:dot-goal-trace" } as never);
expect(deps.workflow!.start).toHaveBeenCalledWith(goal.goalId, "req:dot-goal-trace");
});

it("creates one open Goal per conversation for the owner's Dot and starts its workflow", async () => {
const { service, deps } = fixture();
const goal = await service.create("user:1", dotId, input, null);
Expand Down
3 changes: 2 additions & 1 deletion apps/api/tests/memory-consolidation-coordinator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ describe("Memory consolidation coordinator", () => {
taskQueue: MEMORY_CONSOLIDATION_TASK_QUEUE,
workflowId,
workflowIdReusePolicy: "REJECT_DUPLICATE",
workflowIdConflictPolicy: "USE_EXISTING"
workflowIdConflictPolicy: "USE_EXISTING",
memo: { spaceTraceId: input.traceId }
}
}]);
});
Expand Down
3 changes: 2 additions & 1 deletion apps/api/tests/memory-mutation-coordinator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ describe("Memory mutation coordinator", () => {
taskQueue: MEMORY_MUTATION_TASK_QUEUE,
workflowId: result.workflowId,
workflowIdReusePolicy: "REJECT_DUPLICATE",
workflowIdConflictPolicy: "USE_EXISTING"
workflowIdConflictPolicy: "USE_EXISTING",
memo: { spaceTraceId: input.traceId }
}
}
]);
Expand Down
Loading