Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions packages/opencode/src/session/retry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@ export type Retryable = {

export const RETRY_INITIAL_DELAY = 2000
export const RETRY_BACKOFF_FACTOR = 2
export const RETRY_JITTER_FACTOR = 0.25
export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds
export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout
export const RETRY_MAX_RETRIES = 5

const RETRYABLE_MESSAGE_PATTERNS = [
/429|500|502|503|504|524/i,
Expand All @@ -41,7 +43,7 @@ function cap(ms: number) {
return Math.min(ms, RETRY_MAX_DELAY)
}

export function delay(attempt: number, error?: SessionV1.APIError) {
export function delay(attempt: number, error?: SessionV1.APIError, random = Math.random()) {
if (error) {
const headers = error.data.responseHeaders
if (headers) {
Expand All @@ -67,11 +69,16 @@ export function delay(attempt: number, error?: SessionV1.APIError) {
}
}

return cap(RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1))
return cap(exponential(attempt, random))
}
}

return cap(Math.min(RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1), RETRY_MAX_DELAY_NO_HEADERS))
return cap(Math.min(exponential(attempt, random), RETRY_MAX_DELAY_NO_HEADERS))
}

function exponential(attempt: number, random: number) {
const base = RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1)
return Math.ceil(base + base * RETRY_JITTER_FACTOR * random)
}

export function retryable(error: Err, provider: string) {
Expand Down Expand Up @@ -182,6 +189,7 @@ export function policy(opts: {
const error = opts.parse(meta.input)
const retry = retryable(error, opts.provider)
if (!retry) return Cause.done(meta.attempt)
if (meta.attempt > RETRY_MAX_RETRIES) return Cause.done(meta.attempt)
return Effect.gen(function* () {
const wait = delay(meta.attempt, SessionV1.APIError.isInstance(error) ? error : undefined)
const now = yield* Clock.currentTimeMillis
Expand Down
39 changes: 35 additions & 4 deletions packages/opencode/test/session/retry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,18 @@ function wrap(message: unknown): ReturnType<NamedError["toObject"]> {
describe("session.retry.delay", () => {
test("caps delay at 30 seconds when headers missing", () => {
const error = apiError()
const delays = Array.from({ length: 10 }, (_, index) => SessionRetry.delay(index + 1, error))
const delays = Array.from({ length: 10 }, (_, index) => SessionRetry.delay(index + 1, error, 0))
expect(delays).toStrictEqual([2000, 4000, 8000, 16000, 30000, 30000, 30000, 30000, 30000, 30000])
})

test("adds jitter to exponential delays", () => {
const error = apiError()
expect(SessionRetry.delay(1, error, 0)).toBe(2000)
expect(SessionRetry.delay(1, error, 1)).toBe(2500)
expect(SessionRetry.delay(4, error, 1)).toBe(20000)
expect(SessionRetry.delay(5, error, 1)).toBe(30000)
})

test("prefers retry-after-ms when shorter than exponential", () => {
const error = apiError({ "retry-after-ms": "1500" })
expect(SessionRetry.delay(4, error)).toBe(1500)
Expand All @@ -59,18 +67,18 @@ describe("session.retry.delay", () => {

test("ignores invalid retry hints", () => {
const error = apiError({ "retry-after": "not-a-number" })
expect(SessionRetry.delay(1, error)).toBe(2000)
expect(SessionRetry.delay(1, error, 0)).toBe(2000)
})

test("ignores malformed date retry hints", () => {
const error = apiError({ "retry-after": "Invalid Date String" })
expect(SessionRetry.delay(1, error)).toBe(2000)
expect(SessionRetry.delay(1, error, 0)).toBe(2000)
})

test("ignores past date retry hints", () => {
const pastDate = new Date(Date.now() - 5000).toUTCString()
const error = apiError({ "retry-after": pastDate })
expect(SessionRetry.delay(1, error)).toBe(2000)
expect(SessionRetry.delay(1, error, 0)).toBe(2000)
})

test("uses retry-after values even when exceeding 10 minutes with headers", () => {
Expand Down Expand Up @@ -115,6 +123,29 @@ describe("session.retry.delay", () => {
})
}),
)

it.instance("policy stops after five retries", () =>
Effect.gen(function* () {
const attempts: number[] = []
const error = apiError({ "retry-after-ms": "0" })
const step = yield* Schedule.toStepWithMetadata(
SessionRetry.policy({
provider: "test",
parse: Schema.decodeUnknownSync(SessionV1.APIError.Schema),
set: (info) =>
Effect.sync(() => {
attempts.push(info.attempt)
}),
}),
)

yield* Effect.forEach(Array.from({ length: SessionRetry.RETRY_MAX_RETRIES + 1 }), () =>
Effect.ignore(step(error)),
)

expect(attempts).toStrictEqual([1, 2, 3, 4, 5])
}),
)
})

describe("session.retry.retryable", () => {
Expand Down
Loading