diff --git a/docs/operations/2026-08-05-availability-probes.md b/docs/operations/2026-08-05-availability-probes.md new file mode 100644 index 000000000..a901f3c91 --- /dev/null +++ b/docs/operations/2026-08-05-availability-probes.md @@ -0,0 +1,126 @@ +# Liveness and readiness probe operations + +Date: 2026-08-05 +Maturity: `ACTIVE_PR` until this exact implementation integrates +Canonical decision owner: ADR-0006 in the canonical documentation line + +## Operational problem + +Clearfolio previously exposed only `GET /healthz`. The implementation returned a +static success payload and described the endpoint as a liveness check, while +some deployment prose used the same route as a traffic-readiness signal. That +ambiguity lets an orchestrator route requests to a live-but-unready instance or +restart a recoverable process during a temporary dependency condition. + +Liveness and readiness answer different questions: + +- liveness asks whether the process is irrecoverably unhealthy and should be + restarted; +- readiness asks whether the current instance should receive traffic. + +A temporary readiness failure must not create a restart cascade. + +## Route contract + +This implementation exposes two unauthenticated, non-cacheable probes on the +main application port: + +| Route | Source of truth | Success | Unavailable | +| --- | --- | --- | --- | +| `GET /healthz` | Spring Boot `LivenessState` | `200 {"status":"ok"}` | `503 {"status":"broken"}` | +| `GET /readyz` | Spring Boot `ReadinessState` | `200 {"status":"ready"}` | `503 {"status":"not_ready"}` | + +Both responses include `Cache-Control: no-store`. The successful `/healthz` +payload remains stable for existing callers. + +The liveness probe remains independent of shared external services such as a +database, object store, gateway, or model provider. Readiness may later include +instance-local conditions that determine whether this instance can safely +accept traffic, such as completed startup recovery or bounded-queue overload. +Any extension must update the canonical availability decision and add +executable failure and recovery tests. + +Spring Boot `ApplicationAvailability` is the in-process source of truth. The +controller fails construction when that provider is absent rather than +inventing healthy state. + +## Kubernetes example + +A startup probe protects variable initialization from premature liveness +restarts. Kubernetes suppresses liveness and readiness checks until the startup +probe succeeds. + +```yaml +startupProbe: + httpGet: + path: /healthz + port: 8080 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 24 +livenessProbe: + httpGet: + path: /healthz + port: 8080 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 +readinessProbe: + httpGet: + path: /readyz + port: 8080 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 3 + successThreshold: 1 +``` + +Probe timings are deployment inputs rather than application constants. +Operators tune startup budgets, periods, timeouts, and failure thresholds +against measured startup, overload, and recovery behavior. + +## Security and privacy + +- Probe responses disclose only one controlled state label. +- They contain no tenant, document, queue, dependency, credential, build, or + exception detail. +- They are intentionally unauthenticated so orchestrators can call them, but + they grant no access to protected APIs. +- `Cache-Control: no-store` prevents stale availability replay. +- Responses remain deliberately small. + +## Verification contract + +Exact-head tests prove: + +- `CORRECT` liveness returns `200` and `ok`; +- `BROKEN` liveness returns `503` and `broken`; +- `ACCEPTING_TRAFFIC` readiness returns `200` and `ready`; +- `REFUSING_TRAFFIC` readiness returns `503` and `not_ready`; +- every probe response is non-cacheable; +- controller construction fails without the availability provider; +- production line and branch coverage remains 100%. + +Release and merge still require current exact-head CI, Security Scan, SAST, +fuzzing, zero valid unresolved findings, qualifying independent approval, and +all repository protections. A source-head success on an older base is not +current integration evidence. + +## Rollback + +A rollback may remove `/readyz` and restore the old `/healthz` implementation, +but deployment manifests must be rolled back at the same time. Do not point +both Kubernetes probes at `/healthz`, because doing so recreates the original +semantic ambiguity. Remove the startup probe only when measured startup +behavior and the replacement deployment policy provide an equivalent +startup-failure budget. + +## References + +Broadcom, Inc. (n.d.). *SpringApplication: Application availability (Spring Boot +3.5.16)*. Spring. Retrieved August 5, 2026, from +https://docs.spring.io/spring-boot/3.5/reference/features/spring-application.html#features.spring-application.application-availability + +The Kubernetes Authors. (2026, April 17). *Liveness, readiness, and startup +probes*. Kubernetes. +https://kubernetes.io/docs/concepts/workloads/pods/probes/ diff --git a/src/main/java/com/clearfolio/viewer/controller/HealthController.java b/src/main/java/com/clearfolio/viewer/controller/HealthController.java index 9975524ce..dcdca47b8 100644 --- a/src/main/java/com/clearfolio/viewer/controller/HealthController.java +++ b/src/main/java/com/clearfolio/viewer/controller/HealthController.java @@ -1,36 +1,81 @@ package com.clearfolio.viewer.controller; import java.util.Map; +import java.util.Objects; +import org.springframework.boot.availability.ApplicationAvailability; +import org.springframework.boot.availability.LivenessState; +import org.springframework.boot.availability.ReadinessState; +import org.springframework.http.CacheControl; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; /** - * Lightweight endpoint used for process-liveness checks. + * Exposes separate liveness and readiness probes on the application port. * - *

This endpoint deliberately reports only whether the application process can - * answer requests. Traffic-readiness semantics are introduced separately so an - * orchestrator never confuses restart eligibility with dependency readiness.

+ *

Liveness answers whether this process can continue operating or needs a + * restart. Readiness answers whether the instance should receive traffic. The + * two signals deliberately remain separate so a temporary readiness failure + * does not trigger a restart cascade.

*/ @RestController -@RequestMapping("/healthz") public class HealthController { + private final ApplicationAvailability applicationAvailability; + /** - * Creates the stateless liveness controller. + * Creates the probe controller from Spring Boot's availability state. + * + * @param applicationAvailability current application availability provider */ - public HealthController() { - // No mutable state or external dependency belongs in the liveness path. + public HealthController(ApplicationAvailability applicationAvailability) { + this.applicationAvailability = Objects.requireNonNull( + applicationAvailability, + "applicationAvailability" + ); } /** - * Returns a static health payload when the service is alive. + * Returns the process liveness state. * - * @return health status payload + * @return {@code 200} with {@code status=ok} when the process can recover, + * otherwise {@code 503} with {@code status=broken} */ - @GetMapping - public Map health() { - return Map.of("status", "ok"); + @GetMapping("/healthz") + public ResponseEntity> liveness() { + return availabilityResponse( + applicationAvailability.getLivenessState() == LivenessState.CORRECT, + "ok", + "broken" + ); + } + + /** + * Returns whether this instance is ready to accept traffic. + * + * @return {@code 200} with {@code status=ready} while accepting traffic, + * otherwise {@code 503} with {@code status=not_ready} + */ + @GetMapping("/readyz") + public ResponseEntity> readiness() { + return availabilityResponse( + applicationAvailability.getReadinessState() == ReadinessState.ACCEPTING_TRAFFIC, + "ready", + "not_ready" + ); + } + + private static ResponseEntity> availabilityResponse( + boolean available, + String availableStatus, + String unavailableStatus + ) { + HttpStatus responseStatus = available ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE; + String statusValue = available ? availableStatus : unavailableStatus; + return ResponseEntity.status(responseStatus) + .cacheControl(CacheControl.noStore()) + .body(Map.of("status", statusValue)); } } diff --git a/src/test/java/com/clearfolio/viewer/controller/HealthControllerTest.java b/src/test/java/com/clearfolio/viewer/controller/HealthControllerTest.java index 4a8be8f76..f6cc1b7a0 100644 --- a/src/test/java/com/clearfolio/viewer/controller/HealthControllerTest.java +++ b/src/test/java/com/clearfolio/viewer/controller/HealthControllerTest.java @@ -1,19 +1,100 @@ package com.clearfolio.viewer.controller; -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import org.junit.jupiter.api.Test; +import org.springframework.boot.availability.ApplicationAvailability; +import org.springframework.boot.availability.LivenessState; +import org.springframework.boot.availability.ReadinessState; +import org.springframework.test.web.reactive.server.WebTestClient; +/** + * Verifies that liveness and readiness expose different operational states. + */ class HealthControllerTest { @Test - void healthControllerReturnsOkPayload() { - final HealthController controller = new HealthController(); + void livenessReturnsOkWhenTheApplicationCanRecover() { + ApplicationAvailability availability = availability( + LivenessState.CORRECT, + ReadinessState.ACCEPTING_TRAFFIC + ); + + client(availability).get() + .uri("/healthz") + .exchange() + .expectStatus().isOk() + .expectHeader().valueEquals("Cache-Control", "no-store") + .expectBody(String.class) + .isEqualTo("{\"status\":\"ok\"}"); + } + + @Test + void livenessReturnsServiceUnavailableForAnUnrecoverableApplication() { + ApplicationAvailability availability = availability( + LivenessState.BROKEN, + ReadinessState.ACCEPTING_TRAFFIC + ); - final Map response = controller.health(); + client(availability).get() + .uri("/healthz") + .exchange() + .expectStatus().isEqualTo(503) + .expectHeader().valueEquals("Cache-Control", "no-store") + .expectBody(String.class) + .isEqualTo("{\"status\":\"broken\"}"); + } + + @Test + void readinessReturnsOkOnlyWhileTrafficCanBeAccepted() { + ApplicationAvailability availability = availability( + LivenessState.CORRECT, + ReadinessState.ACCEPTING_TRAFFIC + ); + + client(availability).get() + .uri("/readyz") + .exchange() + .expectStatus().isOk() + .expectHeader().valueEquals("Cache-Control", "no-store") + .expectBody(String.class) + .isEqualTo("{\"status\":\"ready\"}"); + } + + @Test + void readinessReturnsServiceUnavailableWhileTrafficIsRefused() { + ApplicationAvailability availability = availability( + LivenessState.CORRECT, + ReadinessState.REFUSING_TRAFFIC + ); + + client(availability).get() + .uri("/readyz") + .exchange() + .expectStatus().isEqualTo(503) + .expectHeader().valueEquals("Cache-Control", "no-store") + .expectBody(String.class) + .isEqualTo("{\"status\":\"not_ready\"}"); + } + + @Test + void controllerRejectsMissingAvailabilityStateProvider() { + assertThrows(NullPointerException.class, () -> new HealthController(null)); + } + + private static ApplicationAvailability availability( + LivenessState livenessState, + ReadinessState readinessState + ) { + ApplicationAvailability availability = mock(ApplicationAvailability.class); + when(availability.getLivenessState()).thenReturn(livenessState); + when(availability.getReadinessState()).thenReturn(readinessState); + return availability; + } - assertThat(response).containsEntry("status", "ok"); + private static WebTestClient client(ApplicationAvailability availability) { + return WebTestClient.bindToController(new HealthController(availability)).build(); } }