[Wasm] Recover invalid cached boot resources

src/mono/browser/runtime/loader/assets.ts

@@ -436,6 +436,9 @@ export async function start_asset_download (asset: AssetEntryInternal): Promise<
             // we will not re-try if disabled
             throw err;
         }
+        if (asset.cacheReloadAttempted) {
+            throw err;
+        }
         if (ENVIRONMENT_IS_SHELL || ENVIRONMENT_IS_NODE) {
             // we will not re-try on shell
             throw err;
@@ -552,6 +555,9 @@ async function start_asset_download_sources (asset: AssetEntryInternal): Promise
             asset.pendingDownloadInternal = loadingResource;
             response = await loadingResource.response;
             if (!response || !response.ok) {
+                if (asset.cacheReloadAttempted) {
+                    break;
+                }
                 continue;// next source
             }
             return response;
@@ -564,6 +570,9 @@ async function start_asset_download_sources (asset: AssetEntryInternal): Promise
                     statusText: "" + err,
                 } as any;
             }
+            if (asset.cacheReloadAttempted) {
+                break;
+            }
             continue; //next source
         }
     }
@@ -685,6 +694,26 @@ function fetchResource (asset: AssetEntryInternal): Promise<Response> {
         }
     }
 
+    return fetchResourceWithCacheRecovery(asset, url, fetchOptions);
+}
+
+async function fetchResourceWithCacheRecovery (asset: AssetEntryInternal, url: string, fetchOptions: RequestInit): Promise<Response> {
+    try {
+        const response = await loaderHelpers.fetch_like(url, fetchOptions);
+        if (fetchOptions.cache !== "force-cache" ||
+            asset.cacheReloadAttempted ||
+            (response.ok && response.headers.get("Content-Length") !== "0")) {
+            return response;
+        }
+    } catch (err) {
+        if (fetchOptions.cache !== "force-cache" || asset.cacheReloadAttempted) {
+            throw err;
+        }
+    }
+
+    mono_log_debug(() => `Retrying download '${asset.name}' with cache reload`);
+    asset.cacheReloadAttempted = true;
+    fetchOptions.cache = "reload";
     return loaderHelpers.fetch_like(url, fetchOptions);
 }
 

src/mono/browser/runtime/types/internal.ts

@@ -112,6 +112,7 @@ export interface AssetEntryInternal extends AssetEntry {
     // this could have multiple values in time, because of re-try download logic
     pendingDownloadInternal?: LoadingResource
     cache?: RequestCache
+    cacheReloadAttempted?: boolean
     useCredentials?: boolean
     isCore?: boolean
 }

src/mono/wasm/Wasm.Build.Tests/Blazor/AssetCachingTests.cs

@@ -2,9 +2,11 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using Microsoft.Playwright;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -14,11 +16,181 @@ namespace Wasm.Build.Tests.Blazor;
 
 public class AssetCachingTests : BlazorWasmTestBase
 {
+    private const string InjectClientAssemblyFailureEnvironmentVariable = "BLAZOR_WEB_WASM_INJECT_CLIENT_ASSEMBLY_FAILURE";
+    private const string ClientAssemblyFailureModeEnvironmentVariable = "BLAZOR_WEB_WASM_CLIENT_ASSEMBLY_FAILURE_MODE";
+
     public AssetCachingTests(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext)
         : base(output, buildContext)
     {
     }
 
+    [Fact, TestCategory("no-workload")]
+    public async Task BlazorApp_RecoversFromCachedEmptyFingerprintAsset()
+    {
+        var project = CopyTestAsset(
+            Configuration.Release,
+            aot: false,
+            TestAsset.BlazorWebWasm,
+            "CachedEmptyFingerprintAssetTest",
+            appendUnicodeToPath: false
+        );
+
+        var publishOptions = new PublishOptions(
+            AssertAppBundle: false,
+            ExtraBuildEnvironmentVariables: new Dictionary<string, string>
+            {
+                ["WasmFingerprintAssets"] = "true",
+            });
+        (string projectDir, _) = BlazorPublish(project, Configuration.Release, publishOptions);
+
+        var buildPaths = GetBuildPaths(Configuration.Release, forPublish: true, projectDir: projectDir);
+        var publishedAppPath = Path.Combine(buildPaths.BinDir, "publish");
+        var frameworkPath = Path.Combine(publishedAppPath, "wwwroot", "_framework");
+        var clientAssemblyName = Path.GetFileName(Assert.Single(
+            Directory.EnumerateFiles(frameworkPath, "BlazorWebWasm.Client*.wasm")));
+
+        var counterLoaded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var runOptions = new BlazorRunOptions(Configuration.Release)
+        {
+            BrowserPath = $"cache-seed?asset={Uri.EscapeDataString(clientAssemblyName)}",
+            OnConsoleMessage = (type, msg) =>
+            {
+                if (msg.Contains("Counter.OnAfterRender"))
+                    counterLoaded.TrySetResult();
+            },
+            Test = async (page) =>
+            {
+                await page.WaitForFunctionAsync("document.title === 'cache-seeded'");
+
+                var baseUrl = new Uri(page.Url).GetLeftPart(UriPartial.Authority);
+                using var requestLogClient = new BlazorWebWasmLogClient(baseUrl);
+                var seedLogs = await requestLogClient.GetRequestLogsAsync();
+                var failedRequest = Assert.Single(seedLogs.Where(log => log.WasFailureInjected));
+
+                await page.GotoAsync($"{baseUrl}/counter");
+                await counterLoaded.Task.WaitAsync(TimeSpan.FromSeconds(30));
+
+                var requestLogs = await requestLogClient.GetRequestLogsAsync();
+                var clientAssemblyRequests = requestLogs
+                    .Where(log => string.Equals(log.Path, failedRequest.Path, StringComparison.Ordinal))
+                    .ToArray();
+
+                Assert.Equal(2, clientAssemblyRequests.Length);
+                Assert.All(clientAssemblyRequests, log => Assert.Equal(200, log.StatusCode));
+                Assert.True(clientAssemblyRequests[0].WasFailureInjected);
+                Assert.False(clientAssemblyRequests[1].WasFailureInjected);
+
+                await requestLogClient.ClearRequestLogsAsync();
+                counterLoaded = new(TaskCreationOptions.RunContinuationsAsynchronously);
+                await page.ReloadAsync();
+                await counterLoaded.Task.WaitAsync(TimeSpan.FromSeconds(30));
+
+                requestLogs = await requestLogClient.GetRequestLogsAsync();
+                Assert.DoesNotContain(requestLogs, log => string.Equals(log.Path, failedRequest.Path, StringComparison.Ordinal));
+
+                await page.EvaluateAsync("console.log('WASM EXIT 0');");
+            }
+        };
+
+        var publishedAppDllFileName = $"{project.ProjectName}.dll";
+        using ToolCommand cmd = new DotNetCommand(s_buildEnv, _testOutput)
+            .WithWorkingDirectory(publishedAppPath)
+            .WithEnvironmentVariable(InjectClientAssemblyFailureEnvironmentVariable, "true");
+        await BrowserRun(cmd, $"exec {publishedAppDllFileName}", runOptions);
+    }
+
+    [Theory, TestCategory("no-workload")]
+    [InlineData("empty", 200)]
+    [InlineData("error", 503)]
+    [InlineData("integrity", 200)]
+    public Task BlazorApp_DoesNotRetryAfterFailedCacheRecovery(string failureMode, int expectedStatusCode)
+        => RunFailedCacheRecoveryTest(
+            fingerprintAssets: true,
+            failureMode: failureMode,
+            expectedStatusCode: expectedStatusCode,
+            expectedRequestCount: 2);
+
+    [Fact, TestCategory("no-workload")]
+    public Task BlazorApp_DoesNotUseCacheRecoveryForNonForceCache()
+        => RunFailedCacheRecoveryTest(
+            fingerprintAssets: false,
+            failureMode: "empty",
+            expectedStatusCode: 200,
+            expectedRequestCount: 1);
+
+    private async Task RunFailedCacheRecoveryTest(
+        bool fingerprintAssets,
+        string failureMode,
+        int expectedStatusCode,
+        int expectedRequestCount)
+    {
+        var project = CopyTestAsset(
+            Configuration.Release,
+            aot: false,
+            TestAsset.BlazorWebWasm,
+            $"CacheRecoveryLimitTest_{fingerprintAssets}_{failureMode}",
+            appendUnicodeToPath: false
+        );
+
+        var publishOptions = new PublishOptions(
+            AssertAppBundle: false,
+            ExtraBuildEnvironmentVariables: new Dictionary<string, string>
+            {
+                ["WasmFingerprintAssets"] = fingerprintAssets.ToString(),
+            });
+        (string projectDir, _) = BlazorPublish(project, Configuration.Release, publishOptions);
+
+        var counterLoaded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var startupFailed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var runOptions = new BlazorRunOptions(Configuration.Release)
+        {
+            BrowserPath = "counter",
+            OnConsoleMessage = (type, msg) =>
+            {
+                if (msg.Contains("Counter.OnAfterRender"))
+                    counterLoaded.TrySetResult();
+                if (type == "error" &&
+                    msg.Contains("Error in mono_download_assets:", StringComparison.Ordinal) &&
+                    msg.Contains("BlazorWebWasm.Client", StringComparison.OrdinalIgnoreCase))
+                {
+                    startupFailed.TrySetResult();
+                }
+            },
+            Test = async (page) =>
+            {
+                await startupFailed.Task.WaitAsync(TimeSpan.FromSeconds(30));
+
+                using var requestLogClient = new BlazorWebWasmLogClient(page.Url);
+                var requestLogs = await requestLogClient.GetRequestLogsAsync();
+                var failedRequests = requestLogs.Where(log => log.WasFailureInjected).ToArray();
+                Assert.Equal(expectedRequestCount, failedRequests.Length);
+                var failedRequest = failedRequests[0];
+                var clientAssemblyRequests = requestLogs
+                    .Where(log => string.Equals(log.Path, failedRequest.Path, StringComparison.Ordinal))
+                    .ToArray();
+
+                Assert.Equal(expectedRequestCount, clientAssemblyRequests.Length);
+                Assert.All(clientAssemblyRequests, log =>
+                {
+                    Assert.Equal(expectedStatusCode, log.StatusCode);
+                    Assert.True(log.WasFailureInjected);
+                });
+                Assert.False(counterLoaded.Task.IsCompleted);
+
+                await page.EvaluateAsync("console.log('WASM EXIT 0');");
+            }
+        };
+
+        var buildPaths = GetBuildPaths(Configuration.Release, forPublish: true, projectDir: projectDir);
+        var publishedAppPath = Path.Combine(buildPaths.BinDir, "publish");
+        var publishedAppDllFileName = $"{project.ProjectName}.dll";
+        using ToolCommand cmd = new DotNetCommand(s_buildEnv, _testOutput)
+            .WithWorkingDirectory(publishedAppPath)
+            .WithEnvironmentVariable(InjectClientAssemblyFailureEnvironmentVariable, expectedRequestCount.ToString())
+            .WithEnvironmentVariable(ClientAssemblyFailureModeEnvironmentVariable, failureMode);
+        await BrowserRun(cmd, $"exec {publishedAppDllFileName}", runOptions);
+    }
+
     [Fact, TestCategory("no-fingerprinting")]
     public async Task BlazorApp_BasedOnFingerprinting_LoadsWasmAssetsFromCache()
     {

src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorWebWasmRequestLog.cs

@@ -11,5 +11,6 @@ public record BlazorWebWasmRequestLog(
     DateTime Timestamp,
     string Method,
     string Path,
-    int StatusCode
+    int StatusCode,
+    bool WasFailureInjected
 );

src/mono/wasm/testassets/BlazorWebWasm/BlazorWebWasm/BlazorWebWasmRequestLog.cs

@@ -2,5 +2,6 @@ public record BlazorWebWasmRequestLog(
     DateTime Timestamp,
     string Method,
     string Path,
-    int StatusCode
+    int StatusCode,
+    bool WasFailureInjected
 );

src/mono/wasm/testassets/BlazorWebWasm/BlazorWebWasm/Program.cs

@@ -12,15 +12,65 @@
 
 var requestLogs = new List<BlazorWebWasmRequestLog>();
 var requestLogsLock = new Lock();
+var injectClientAssemblyFailureValue = Environment.GetEnvironmentVariable("BLAZOR_WEB_WASM_INJECT_CLIENT_ASSEMBLY_FAILURE");
+var remainingClientAssemblyFailures = int.TryParse(injectClientAssemblyFailureValue, out var failureCount)
+    ? failureCount
+    : string.Equals(injectClientAssemblyFailureValue, "true", StringComparison.OrdinalIgnoreCase) ? 1 : 0;
+var clientAssemblyFailureMode = Environment.GetEnvironmentVariable("BLAZOR_WEB_WASM_CLIENT_ASSEMBLY_FAILURE_MODE") ?? "empty";
+var clientAssemblyFailureLock = new Lock();
 
 app.Use(async (context, next) =>
 {
-    await next.Invoke();
+    var wasFailureInjected = false;
+    if (HttpMethods.IsGet(context.Request.Method) &&
+        context.Request.Path.Value is { } path &&
+        path.Contains("BlazorWebWasm.Client", StringComparison.OrdinalIgnoreCase) &&
+        path.EndsWith(".wasm", StringComparison.OrdinalIgnoreCase))
+    {
+        lock (clientAssemblyFailureLock)
+        {
+            if (remainingClientAssemblyFailures > 0)
+            {
+                remainingClientAssemblyFailures--;
+                wasFailureInjected = true;
+            }
+        }
+    }
+
+    if (wasFailureInjected)
+    {
+        context.Response.ContentType = "application/wasm";
+        context.Response.Headers.CacheControl = "public,max-age=3600";
+        switch (clientAssemblyFailureMode)
+        {
+            case "empty":
+                context.Response.StatusCode = StatusCodes.Status200OK;
+                context.Response.ContentLength = 0;
+                break;
+            case "error":
+                context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
+                context.Response.ContentLength = 0;
+                break;
+            case "integrity":
+                context.Response.StatusCode = StatusCodes.Status200OK;
+                context.Response.ContentLength = 1;
+                await context.Response.Body.WriteAsync(new byte[] { 0 });
+                break;
+            default:
+                throw new InvalidOperationException($"Unknown client assembly failure mode '{clientAssemblyFailureMode}'.");
+        }
+    }
+    else
+    {
+        await next.Invoke();
+    }
+
     var logEntry = new BlazorWebWasmRequestLog(
         DateTime.UtcNow,
         context.Request.Method,
         context.Request.Path,
-        context.Response.StatusCode
+        context.Response.StatusCode,
+        wasFailureInjected
     );
     lock (requestLogsLock)
     {
@@ -63,5 +113,26 @@
         requestLogs.Clear();
     }
 });
+app.MapGet("/cache-seed", () => Results.Content(
+    """
+    <!DOCTYPE html>
+    <html>
+    <body>
+    <main class="page">Seeding cache...</main>
+    <script>
+    (async () => {
+        const assemblyName = new URLSearchParams(location.search).get('asset');
+        if (!assemblyName) {
+            throw new Error('The client assembly was not specified.');
+        }
+
+        await fetch(`/_framework/${assemblyName}`);
+        document.title = 'cache-seeded';
+    })();
+    </script>
+    </body>
+    </html>
+    """,
+    "text/html"));
 
 app.Run();