Cloud Logging library does not work on Java Flex Compat

[meltsufin]

protected void doGet(HttpServletRequest req, HttpServletResponse resp)
      throws ServletException, IOException {
    final PrintWriter out = resp.getWriter();

    LoggingOptions options = LoggingOptions.newBuilder().build();

    try (Logging logging = options.getService()) {


      LogEntry firstEntry = LogEntry
          .newBuilder(Payload.StringPayload.of("testing direct use of API"))
          .setLogName("test-log")
          .setResource(MonitoredResource.newBuilder("global")
              .addLabel("project_id", options.getProjectId())
              .build())
          .build();
      logging.write(Collections.singleton(firstEntry));

      Page<LogEntry> entries = logging.listLogEntries(
          Logging.EntryListOption
              .filter("logName=projects/" + options.getProjectId() + "/logs/test-log"));
      Iterator<LogEntry> entryIterator = entries.iterateAll();
      while (entryIterator.hasNext()) {
        System.out.println(entryIterator.next());
      }


    } catch (Exception e) {
      e.printStackTrace(out);
      e.printStackTrace();
    }
}

com.google.cloud.logging.LoggingException: io.grpc.StatusRuntimeException: UNKNOWN
	at com.google.cloud.logging.spi.DefaultLoggingRpc$1.apply(DefaultLoggingRpc.java:165)
	at com.google.cloud.logging.spi.DefaultLoggingRpc$1.apply(DefaultLoggingRpc.java:159)
	at com.google.api.gax.grpc.AbstractRpcFuture$2.apply(AbstractRpcFuture.java:65)
	at com.google.api.gax.grpc.AbstractRpcFuture$2.apply(AbstractRpcFuture.java:62)
	at com.google.common.util.concurrent.Futures$CatchingFuture.doFallback(Futures.java:842)
	at com.google.common.util.concurrent.Futures$CatchingFuture.doFallback(Futures.java:834)
	at com.google.common.util.concurrent.Futures$AbstractCatchingFuture.run(Futures.java:789)
	at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
	at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
	at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
	at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:634)
	at com.google.common.util.concurrent.SettableFuture.setException(SettableFuture.java:53)
	at com.google.api.gax.grpc.BundlingFuture.setException(BundlingFuture.java:87)
	at com.google.api.gax.grpc.BundlingContext.sendResult(BundlingContext.java:97)
	at com.google.api.gax.grpc.BundleExecutor.processBundle(BundleExecutor.java:89)
	at com.google.api.gax.bundling.ThresholdBundlingForwarder$BundleForwardingRunnable.processBundle(ThresholdBundlingForwarder.java:106)
	at com.google.api.gax.bundling.ThresholdBundlingForwarder$BundleForwardingRunnable.run(ThresholdBundlingForwarder.java:89)
	at java.lang.Thread.run(Thread.java:745)
Caused by: com.google.api.gax.grpc.ApiException: io.grpc.StatusRuntimeException: UNKNOWN
	at com.google.api.gax.grpc.ExceptionTransformingCallable$ExceptionTransformingFuture.onFailure(ExceptionTransformingCallable.java:107)
	at com.google.api.gax.grpc.ListenableFutureDelegate$1.onFailure(ListenableFutureDelegate.java:52)
	at com.google.common.util.concurrent.Futures$6.run(Futures.java:1764)
	at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
	at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
	at com.google.common.util.concurrent.AbstractFuture.addListener(AbstractFuture.java:595)
	at com.google.common.util.concurrent.ForwardingListenableFuture.addListener(ForwardingListenableFuture.java:47)
	at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1776)
	at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1713)
	at com.google.api.gax.grpc.ListenableFutureDelegate.addCallback(ListenableFutureDelegate.java:47)
	at com.google.api.gax.grpc.ExceptionTransformingCallable.futureCall(ExceptionTransformingCallable.java:63)
	at com.google.api.gax.grpc.RetryingCallable$RetryingResultFuture.issueCall(RetryingCallable.java:214)
	at com.google.api.gax.grpc.RetryingCallable.futureCall(RetryingCallable.java:82)
	at com.google.api.gax.grpc.UnaryCallable.futureCall(UnaryCallable.java:243)
	at com.google.api.gax.grpc.UnaryCallable.futureCall(UnaryCallable.java:254)
	at com.google.api.gax.grpc.UnaryCallable.call(UnaryCallable.java:282)
	at com.google.api.gax.grpc.BundleExecutor.processBundle(BundleExecutor.java:82)
	... 3 more
Caused by: io.grpc.StatusRuntimeException: UNKNOWN
	at io.grpc.Status.asRuntimeException(Status.java:545)
	at io.grpc.stub.ClientCalls$UnaryStreamToFuture.onClose(ClientCalls.java:442)
	at io.grpc.ClientInterceptors$CheckedForwardingClientCall.start(ClientInterceptors.java:203)
	at io.grpc.ForwardingClientCall.start(ForwardingClientCall.java:47)
	at com.google.api.gax.grpc.HeaderInterceptor$1.start(HeaderInterceptor.java:62)
	at io.grpc.stub.ClientCalls.startCall(ClientCalls.java:273)
	at io.grpc.stub.ClientCalls.asyncUnaryRequestCall(ClientCalls.java:252)
	at io.grpc.stub.ClientCalls.futureUnaryCall(ClientCalls.java:189)
	at com.google.api.gax.grpc.DirectCallable.futureCall(DirectCallable.java:58)
	at com.google.api.gax.grpc.ExceptionTransformingCallable.futureCall(ExceptionTransformingCallable.java:60)
	... 9 more
Caused by: java.lang.IllegalStateException: Request threads can only be created within the context of a running request.
	at com.google.appengine.repackaged.com.google.common.base.Preconditions.checkState(Preconditions.java:446)
	at com.google.apphosting.vmruntime.VmRequestThreadFactory.newThread(VmRequestThreadFactory.java:79)
	at com.google.common.util.concurrent.ThreadFactoryBuilder$1.newThread(ThreadFactoryBuilder.java:162)
	at java.util.concurrent.ThreadPoolExecutor$Worker.<init>(ThreadPoolExecutor.java:612)
	at java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:925)
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
	at io.grpc.internal.DnsNameResolver.resolve(DnsNameResolver.java:202)
	at io.grpc.internal.DnsNameResolver.start(DnsNameResolver.java:120)
	at io.grpc.internal.ManagedChannelImpl$1NameResolverStartTask.run(ManagedChannelImpl.java:268)
	at io.grpc.internal.ManagedChannelImpl.exitIdleModeAndGetLb(ManagedChannelImpl.java:233)
	at io.grpc.internal.ManagedChannelImpl$2.get(ManagedChannelImpl.java:310)
	at io.grpc.internal.ClientCallImpl.start(ClientCallImpl.java:213)
	at io.grpc.auth.ClientAuthInterceptor$1.checkedStart(ClientAuthInterceptor.java:104)
	at io.grpc.ClientInterceptors$CheckedForwardingClientCall.start(ClientInterceptors.java:195)
	... 16 more

Note, I'm using a custom Dockerfile:

FROM gcr.io/google-appengine/jetty9-compat:githubheadasync
ADD . /app/

I think the problem stems from the library trying to use com.google.apphosting.vmruntime.VmRequestThreadFactory that is configured in the compat runtime.

GRPC is checking for the com.google.appengine.runtime.environment system property and is going into the App Engine branch.

Interestingly, if I clear the system property with System.clearProperty("com.google.appengine.runtime.environment"), logging seems to work, but that's just a hack.

@garrettjonesgoogle @ludoch @gregw

[ludoch]

We should also test it on GAE Standard (Java7 and Java8) and fix whatever issues we see there...
Who is supposed to test cloud-java on GAE standard?

[lesv]

We should figure out the right way to check and get everyone on the same page.

[gregw]

Should I move forward on the branch GoogleCloudPlatform/jetty-runtime#81 to add in testing for remote logging - ie were we have tests that should log and use the API to verify the logs were received OK?

[meltsufin]

@gregw I think on the jetty-runtime side, we should make sure that our default logging configuration works for sending logs to Stackdriver with the metadata. However, I don't think it's a good place for testing just whether the logging API works on Flex.

[garrettjonesgoogle]

Does this block GA?

[lesv]

Mike can say for sure, but I think so.

[meltsufin]

Yes, I would say so.