Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 39 additions & 5 deletions src/main/java/org/unicitylabs/sdk/api/BatchRequestOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
import org.unicitylabs.sdk.crypto.hash.HashAlgorithm;
import org.unicitylabs.sdk.internal.OperationContext;

import java.time.Duration;
import java.util.Objects;

/**
* Options of {@link BatchAggregatorClient#getInclusionProofs}.
*
Expand All @@ -13,16 +16,19 @@
*/
public final class BatchRequestOptions {

private static final BatchRequestOptions LATEST = new BatchRequestOptions(null, null, null);
private static final BatchRequestOptions LATEST = new BatchRequestOptions(null, null, null, null);

private final DataHash stateRoot;
private final Long waitMillis;
private final OperationContext operation;
private final Duration callTimeout;

private BatchRequestOptions(DataHash stateRoot, Long waitMillis, OperationContext operation) {
private BatchRequestOptions(DataHash stateRoot, Long waitMillis, OperationContext operation,
Duration callTimeout) {
this.stateRoot = stateRoot;
this.waitMillis = waitMillis;
this.operation = operation;
this.callTimeout = callTimeout;
}

/**
Expand All @@ -45,7 +51,7 @@ public static BatchRequestOptions exact(DataHash stateRoot) {
if (stateRoot == null || stateRoot.getAlgorithm() != HashAlgorithm.SHA256) {
throw new IllegalArgumentException("State root must be a SHA-256 hash.");
}
return new BatchRequestOptions(stateRoot, null, null);
return new BatchRequestOptions(stateRoot, null, null, null);
}

/**
Expand All @@ -65,7 +71,7 @@ public BatchRequestOptions withWaitMillis(long waitMillis) {
if (waitMillis < 0 || waitMillis > WaitOptions.MAX_WAIT_MILLIS) {
throw new IllegalArgumentException("waitMillis must be in 0.." + WaitOptions.MAX_WAIT_MILLIS);
}
return new BatchRequestOptions(null, waitMillis, operation);
return new BatchRequestOptions(null, waitMillis, operation, callTimeout);
}

/**
Expand All @@ -75,7 +81,26 @@ public BatchRequestOptions withWaitMillis(long waitMillis) {
* @return new options
*/
public BatchRequestOptions withOperation(OperationContext operation) {
return new BatchRequestOptions(stateRoot, waitMillis, operation);
return new BatchRequestOptions(stateRoot, waitMillis, operation, callTimeout);
}

/**
* Bounds one HTTP call, in addition to the budget of the operation. A call that is held by the
* aggregator needs a timeout of its wait plus a response margin, because the HTTP client's own
* read timeout is shorter.
*
* @param callTimeout positive duration, at most {@link RequestOptions#MAX_CALL_TIMEOUT}
* @return new options
* @throws IllegalArgumentException if the duration is not positive or above the limit
*/
public BatchRequestOptions withCallTimeout(Duration callTimeout) {
Objects.requireNonNull(callTimeout, "callTimeout cannot be null");
if (callTimeout.isNegative() || callTimeout.isZero()
|| callTimeout.compareTo(RequestOptions.MAX_CALL_TIMEOUT) > 0) {
throw new IllegalArgumentException("callTimeout must be positive and at most "
+ RequestOptions.MAX_CALL_TIMEOUT.getSeconds() + " s (MAX_CALL_TIMEOUT).");
}
return new BatchRequestOptions(stateRoot, waitMillis, operation, callTimeout);
}

/**
Expand Down Expand Up @@ -104,4 +129,13 @@ public Long getWaitMillis() {
public OperationContext getOperation() {
return operation;
}

/**
* Timeout of one HTTP call.
*
* @return duration, null when the transport default applies
*/
public Duration getCallTimeout() {
return callTimeout;
}
}
26 changes: 21 additions & 5 deletions src/main/java/org/unicitylabs/sdk/api/JsonRpcAggregatorClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.unicitylabs.sdk.util.HexConverter;

import java.net.URI;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
Expand Down Expand Up @@ -304,6 +305,9 @@ public CompletableFuture<InclusionProofBatch> getInclusionProofs(List<StateId> s
if (options.getOperation() != null) {
callOptions = callOptions.withOperation(options.getOperation());
}
if (options.getCallTimeout() != null) {
callOptions = callOptions.withCallTimeout(options.getCallTimeout());
}
if (waitMillis != null) {
callOptions = callOptions.withWaitMillis(waitMillis);
}
Expand Down Expand Up @@ -333,6 +337,7 @@ private final class BatchCall {
private final Long waitMillis;
private final RequestOptions options;
private volatile CompletableFuture<JsonNode> inFlight;
private volatile long startedNanos;

BatchCall(List<String> ids, String stateRoot, boolean exact, Long waitMillis,
RequestOptions options) {
Expand All @@ -351,6 +356,7 @@ void cancel() {
}

void start(CompletableFuture<InclusionProofBatch> result) {
this.startedNanos = System.nanoTime();
this.send(this.waitMillis, 1, result);
}

Expand All @@ -362,8 +368,21 @@ private void send(Long wait, int requests, CompletableFuture<InclusionProofBatch
if (wait != null) {
params.put("waitMs", wait);
}
RequestOptions callOptions = wait == null && this.waitMillis != null
? withoutWait(this.options) : this.options;
RequestOptions callOptions = this.options;
if (wait == null && this.waitMillis != null) {
callOptions = withoutWait(this.options);
Duration bound = this.options.getCallTimeout();
if (bound != null) {
// the repeat shares the one call bound of the request: only the time left is used
Duration left = bound.minusNanos(System.nanoTime() - this.startedNanos);
if (left.isZero() || left.isNegative()) {
result.completeExceptionally(batchFailure(new java.net.SocketTimeoutException(
"The call timeout expired before the repeat without waitMs."), requests, null));
return;
}
callOptions = callOptions.withCallTimeout(left);
}
}
CompletableFuture<JsonNode> call = JsonRpcAggregatorClient.this.transport
.call("get_inclusion_proofs.v1", params, JsonRpcAggregatorClient.this.authHeaders(),
callOptions);
Expand Down Expand Up @@ -420,9 +439,6 @@ private static RequestOptions withoutWait(RequestOptions options) {
if (options.getOperation() != null) {
copy = copy.withOperation(options.getOperation());
}
if (options.getCallTimeout() != null) {
copy = copy.withCallTimeout(options.getCallTimeout());
}
return copy;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.unicitylabs.sdk.api.InclusionProofBatch;
import org.unicitylabs.sdk.api.InclusionProofResponse;
import org.unicitylabs.sdk.api.LeafValue;
import org.unicitylabs.sdk.api.RequestOptions;
import org.unicitylabs.sdk.api.StateId;
import org.unicitylabs.sdk.api.bft.AggregationPartition;
import org.unicitylabs.sdk.api.bft.AuthenticatedPartitionView;
Expand Down Expand Up @@ -663,17 +664,20 @@ private CompletableFuture<ShardBatch> fetch(Group group, List<StateId> stateIds,
requestOptions = BatchRequestOptions.latest();
long wait = master ? masterWait() : 0;
if (wait > 0) {
requestOptions = requestOptions.withWaitMillis(wait);
requestOptions = requestOptions.withWaitMillis(wait)
.withCallTimeout(callTimeout(wait));
}
}
// only the call that waits carries a call bound of its own: only its expiry is a timeout
boolean bounded = requestOptions.getCallTimeout() != null;
requestOptions = requestOptions.withOperation(attempt);
BatchAggregatorClient client = (BatchAggregatorClient) group.endpoint.getClient();
CompletableFuture<InclusionProofBatch> call = client.getInclusionProofs(ids, requestOptions);
OperationContext.Registration registration = scope.onCancel(() -> call.cancel(true));
call.whenComplete((batch, error) -> {
registration.remove();
try {
settle(out, group, ids.size(), master, batch, error);
settle(out, group, ids.size(), master, bounded, batch, error);
} catch (Throwable t) {
// whatever goes wrong while classifying a response fails the step, typed and at once
out.completeExceptionally(t);
Expand All @@ -687,17 +691,17 @@ private CompletableFuture<ShardBatch> fetch(Group group, List<StateId> stateIds,
}
});
} catch (Throwable t) {
out.completeExceptionally(requestFailure(t, group.endpoint.getShard(), master,
out.completeExceptionally(requestFailure(t, group.endpoint.getShard(), master, false,
group.indexes));
}
return out;
}

private void settle(CompletableFuture<ShardBatch> out, Group group, int expected,
boolean master, InclusionProofBatch batch, Throwable error) {
boolean master, boolean bounded, InclusionProofBatch batch, Throwable error) {
if (error != null) {
out.completeExceptionally(requestFailure(error, group.endpoint.getShard(), master,
group.indexes));
bounded, group.indexes));
return;
}
if (batch == null || batch.getProofs().size() != expected) {
Expand All @@ -711,6 +715,20 @@ private void settle(CompletableFuture<ShardBatch> out, Group group, int expected
out.complete(new ShardBatch(batch, group));
}

/** The timeout of the call that waits: wait plus margin, within what the attempt has left. */
private Duration callTimeout(long wait) {
Duration timeout = Duration.ofMillis(wait).plus(options.getMargin());
Duration left = attempt.remaining();
if (left.compareTo(timeout) < 0) {
timeout = left;
}
if (timeout.compareTo(RequestOptions.MAX_CALL_TIMEOUT) > 0) {
timeout = RequestOptions.MAX_CALL_TIMEOUT;
}
// a spent budget is not a zero timeout: the attempt deadline ends the call
return timeout.isZero() || timeout.isNegative() ? Duration.ofMillis(1) : timeout;
}

/** The wait that fits the attempt deadline, or none: {@code min(window, left - margin)}. */
private long masterWait() {
if (!options.isMasterLongPoll()) {
Expand Down Expand Up @@ -800,7 +818,7 @@ private void finishFetch(CompletableFuture<List<ShardBatch>> all, ShardBatch[] r

/** Maps whatever a failed call produced to the failure or unavailable outcome of the attempt. */
private Throwable requestFailure(Throwable raised, ShardId shard, boolean master,
List<Integer> positions) {
boolean bounded, List<Integer> positions) {
Throwable e = unwrap(raised);
ConsolidationException done = done();
if (done != null && (e instanceof CancellationException
Expand Down Expand Up @@ -839,11 +857,23 @@ private Throwable requestFailure(Throwable raised, ShardId shard, boolean master
return new ConsolidationException(ConsolidationException.Kind.INVALID_RESPONSE,
batch.getMessage(), ConsolidationOutcome.INVALID_INPUT, detail, false, shard, batch);
default:
return new ConsolidationException(ConsolidationException.Kind.TRANSPORT,
// the master's call that outlived its own bound (wait plus margin) is a retryable timeout;
// a transport timeout of any other call stays a transport failure
return new ConsolidationException(bounded && timedOut(batch)
? ConsolidationException.Kind.TIMEOUT : ConsolidationException.Kind.TRANSPORT,
batch.getMessage(), null, detail, true, shard, batch);
}
}

private boolean timedOut(Throwable e) {
for (int depth = 0; e != null && depth < 8; e = e.getCause(), depth++) {
if (e instanceof java.io.InterruptedIOException) {
return true;
}
}
return false;
}

private UnavailableSignal unavailable(BatchProofException batch, String detail,
boolean master, List<Integer> positions) {
String d = detail == null ? "" : detail;
Expand Down
19 changes: 19 additions & 0 deletions src/test/java/org/unicitylabs/sdk/api/LongPollWireTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,25 @@ void theRetryWithoutTheWaitStaysUnderTheBudgetOfTheOperation() throws Exception
assertInstanceOf(java.util.concurrent.TimeoutException.class, WireTestSupport.failure(future));
}

@Test
void theRetryWithoutTheWaitSharesTheCallBoundAndUsesOnlyTheTimeLeft() throws Exception {
// the first answer takes ~300 ms, the bound is 600 ms: the retry gets ~300 ms, not a fresh 600 ms
server.enqueue(new MockResponse().setHeadersDelay(300, java.util.concurrent.TimeUnit.MILLISECONDS)
.setBody(MALFORMED_PARAMS));
server.enqueue(new MockResponse().setHeadersDelay(5, java.util.concurrent.TimeUnit.SECONDS).setBody("{}"));
long started = System.nanoTime();
CompletableFuture<InclusionProofBatch> future = WireTestSupport.client(server, http, 5).getInclusionProofs(
List.of(WireTestSupport.stateId("ab".repeat(32))),
BatchRequestOptions.latest().withWaitMillis(1000)
.withCallTimeout(java.time.Duration.ofMillis(600)));
BatchProofException e = assertInstanceOf(BatchProofException.class, WireTestSupport.failure(future));
long tookMillis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
assertEquals(BatchProofException.Kind.TRANSPORT, e.getKind());
assertEquals(2, e.getRequests());
// 600 ms bound plus scheduling slack, well under 300 + 600
assertTrue(tookMillis < 850, "took " + tookMillis + " ms");
}

@Test
void cancellingTheOperationCancelsTheRetryInFlight() throws Exception {
org.unicitylabs.sdk.internal.OperationContext operation =
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package org.unicitylabs.sdk.consolidation;

import com.fasterxml.jackson.databind.JsonNode;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.unicitylabs.sdk.api.BatchAggregatorClient;
import org.unicitylabs.sdk.api.BatchProofException;
import org.unicitylabs.sdk.api.BatchRequestOptions;
import org.unicitylabs.sdk.api.InclusionProofBatch;
import org.unicitylabs.sdk.api.StateId;
import org.unicitylabs.sdk.api.bft.RootTrustBase;
import org.unicitylabs.sdk.api.bft.ShardId;
import org.unicitylabs.sdk.api.bft.ShardConfigurationPin;
import org.unicitylabs.sdk.util.HexConverter;
import org.unicitylabs.sdk.api.bft.verification.OperationUnicityCertificateVerifier;
import org.unicitylabs.sdk.api.bft.PartitionViewPolicy;
import org.unicitylabs.sdk.api.bft.verification.UnicityCertificateVerifier;
import org.unicitylabs.sdk.api.bft.verification.VerificationPolicy;

/** A peer's ordinary transport timeout in wave 2 stays a retryable transport failure. */
class PeerTransportTimeoutTest {

@Test
void aPeerTransportTimeoutStaysRetryableAndIsNotReportedAsCancelled() throws Exception {
CorpusSupport.Case c = new CorpusSupport.Case("shards-8");
JsonNode context = c.json("context.json");
JsonNode all = c.json("responses.json");
String masterBits = context.get("master").asText();
Map<String, JsonNode> responses = new HashMap<>();
responses.put(masterBits, all.get("master"));
for (JsonNode peer : all.get("peers")) {
responses.put(peer.get("shard").asText(), peer.get("response"));
}
List<ShardId> scheme = new ArrayList<>();
for (JsonNode bits : context.get("shards")) {
scheme.add(CorpusSupport.shard(bits.asText()));
}
List<CorpusSupport.Call> calls = CorpusSupport.newCalls();
List<ShardEndpoint> endpoints = new ArrayList<>();
int[] peerSeen = {0};
for (JsonNode bits : context.get("participants")) {
String b = bits.asText();
BatchAggregatorClient real = new CorpusSupport.FakeShardClient(b, responses.get(b), calls);
BatchAggregatorClient client = b.equals(masterBits) ? real
: (BatchAggregatorClient) java.lang.reflect.Proxy.newProxyInstance(
BatchAggregatorClient.class.getClassLoader(), new Class<?>[] {BatchAggregatorClient.class},
(proxy, method, args) -> {
if (!method.getName().equals("getInclusionProofs")) {
return method.invoke(real, args);
}
int n = peerSeen[0]++;
CompletableFuture<InclusionProofBatch> f = new CompletableFuture<>();
if (n == 0) {
return f; // in flight until the wave is cancelled
}
f.completeExceptionally(new BatchProofException(BatchProofException.Kind.TRANSPORT,
"timeout", null, null, null, true, 1, null,
new java.net.SocketTimeoutException("read timed out")));
return f;
});
endpoints.add(new ShardEndpoint(CorpusSupport.shard(b), client));
}
List<ShardConfigurationPin> pins = new ArrayList<>();
for (JsonNode config : context.get("shardConfigs")) {
pins.add(new ShardConfigurationPin(CorpusSupport.shard(config.get("shard").asText()),
HexConverter.decode(config.get("hash").asText())));
}
RootTrustBase trustBase = CorpusSupport.trustBase(c.bytes("trust-base.cbor"));
OperationUnicityCertificateVerifier verifier = (OperationUnicityCertificateVerifier)
new UnicityCertificateVerifier().beginOperation(VerificationPolicy.LEGACY_QUORUM,
PartitionViewPolicy.none());
ProofConsolidator consolidator = new ProofConsolidator(new ShardRouting(endpoints, scheme, pins),
trustBase, verifier);
ConsolidationException failure;
try {
consolidator.consolidate(CorpusSupport.evidenceOf(c.json("requests.json")),
ConsolidationOptions.defaults().withMaster(CorpusSupport.shard(masterBits))
.withTimeout(Duration.ofSeconds(20)).withExecutor(Runnable::run)).join();
throw new AssertionError("expected a failure");
} catch (CompletionException e) {
failure = (ConsolidationException) e.getCause();
}
assertEquals(ConsolidationException.Kind.TRANSPORT, failure.getKind(), failure.getMessage());
assertTrue(failure.isRetryable());
assertEquals(2, peerSeen[0], "the first peer was in flight when the second timed out");
}
}
Loading
Loading