Check base images using mirror registry instead of public DockerHub

src/ImageBuilder/Commands/GetStaleImagesCommand.cs

@@ -55,7 +55,11 @@ public override async Task ExecuteAsync()
 
             IEnumerable<Task<SubscriptionImagePaths>> getPathResults =
                 SubscriptionHelper.GetSubscriptionManifests(
-                    Options.SubscriptionOptions.SubscriptionsPath, Options.FilterOptions, _gitService, _manifestJsonService)
+                    Options.SubscriptionOptions.SubscriptionsPath,
+                    Options.FilterOptions,
+                    _gitService,
+                    _manifestJsonService,
+                    manifestOptions => manifestOptions.RegistryOverride = Options.RegistryOverride)
                 .Select(async subscriptionManifest =>
                     new SubscriptionImagePaths
                     {
@@ -87,6 +91,12 @@ private async Task<IEnumerable<string>> GetPathsToRebuildAsync(Models.Subscripti
         {
             ImageArtifactDetails imageArtifactDetails = await GetImageInfoForSubscriptionAsync(subscription, manifest);
 
+            ImageNameResolverForMatrix imageNameResolver = new(
+                Options.BaseImageOverrideOptions,
+                manifest,
+                repoPrefix: null,
+                sourceRepoPrefix: Options.SourceRepoPrefix);
+
             List<string> pathsToRebuild = new();
 
             foreach (RepoInfo repo in manifest.FilteredRepos)
@@ -97,7 +107,8 @@ private async Task<IEnumerable<string>> GetPathsToRebuildAsync(Models.Subscripti
 
                 foreach (PlatformInfo platform in platforms)
                 {
-                    pathsToRebuild.AddRange(await GetPathsToRebuildAsync(manifest, platform, repo, imageArtifactDetails));
+                    pathsToRebuild.AddRange(
+                        await GetPathsToRebuildAsync(manifest, platform, repo, imageArtifactDetails, imageNameResolver));
                 }
             }
 
@@ -109,44 +120,81 @@ private static IEnumerable<PlatformInfo> GetDescendants(PlatformInfo platform, M
                 .Prepend(platform);
 
         private async Task<List<string>> GetPathsToRebuildAsync(
-            ManifestInfo manifest, PlatformInfo platform, RepoInfo repo, ImageArtifactDetails imageArtifactDetails)
+            ManifestInfo manifest,
+            PlatformInfo platform,
+            RepoInfo repo,
+            ImageArtifactDetails imageArtifactDetails,
+            ImageNameResolverForMatrix imageNameResolver)
         {
             string? fromImage = platform.FinalStageFromImage;
             if (fromImage is null)
             {
                 _logger.LogInformation(
-                    $"There is no base image for '{platform.DockerfilePath}'. By default, it is considered up-to-date.");
+                    "Dockerfile {DockerfilePath} has no base image. It is automatically considered up-to-date.",
+                    platform.DockerfilePath);
+
                 return [];
             }
 
-            (PlatformData Platform, ImageData Image)? matchingPlatform = ImageInfoHelper.GetMatchingPlatformData(platform, repo, imageArtifactDetails);
+            (PlatformData Platform, ImageData Image)? matchingPlatform =
+                ImageInfoHelper.GetMatchingPlatformData(platform, repo, imageArtifactDetails);
 
             if (matchingPlatform is null)
             {
-                _logger.LogInformation(
-                    $"WARNING: Image info not found for '{platform.DockerfilePath}'. Adding path to build to be queued anyway.");
+                _logger.LogWarning(
+                    "Image info not found for '{DockerfilePath}'. It will be queued for rebuild.",
+                    platform.DockerfilePath);
+
                 IEnumerable<PlatformInfo> dependentPlatforms = GetDescendants(platform, manifest);
                 return dependentPlatforms.Select(p => p.Model.Dockerfile).ToList();
             }
 
-            fromImage = Options.BaseImageOverrideOptions.ApplyBaseImageOverride(fromImage);
-
-            string currentDigest = await LockHelper.DoubleCheckedLockLookupAsync(_imageDigestsLock, _imageDigests, fromImage,
-                async () =>
-                {
-                    string digest = await _manifestService.Value.GetManifestDigestShaAsync(fromImage, Options.IsDryRun);
-                    return DockerHelper.GetDigestString(DockerHelper.GetRepo(fromImage), digest);
-                });
-
-            bool rebuildImage = matchingPlatform.Value.Platform.BaseImageDigest != currentDigest;
+            // Resolve where to actually fetch the digest from. For external base images this
+            // points to the mirror location in the staging registry; for internal images it is the
+            // original FROM tag. The "public" form is the canonical reference matching what gets
+            // recorded in image-info.json and so is the right repo to use in the digest comparison
+            // string below.
+            string baseImagePullReference = imageNameResolver.GetFromImagePullTag(fromImage);
+            string baseImagePublicReference = imageNameResolver.GetFromImagePublicTag(fromImage);
+
+            // Cache the manifest digest by pull reference. The digest is a function of where we
+            // actually pull bytes from, so the pull reference is the correct cache key.
+            string baseImageManifestDigest =
+                await LockHelper.DoubleCheckedLockLookupAsync(
+                    semaphore: _imageDigestsLock,
+                    dictionary: _imageDigests,
+                    key: baseImagePullReference,
+                    getValue: () =>
+                        // This reaches out to the registry to fetch the digest from the pull
+                        // reference. For external images, this fetches from the mirror.
+                        _manifestService.Value.GetManifestDigestShaAsync(baseImagePullReference, Options.IsDryRun));
+
+            // Build a digest-pinned reference of the form '<public-repo>@sha256:<hex>' (e.g.
+            // 'mcr.microsoft.com/dotnet/runtime@sha256:abc123...'). This must be built per-call
+            // from this platform's own public reference — two FROM spellings (e.g. 'almalinux:8'
+            // vs 'library/almalinux:8') can share a pull reference but resolve to different
+            // public references, so the formed string cannot be cached or shared across platforms.
+            // The shape matches what's stored in Platform.BaseImageDigest so the equality check
+            // below is meaningful.
+            string currentBaseImageDigestReference =
+                DockerHelper.GetDigestString(
+                    repo: DockerHelper.GetRepo(baseImagePublicReference),
+                    sha: baseImageManifestDigest);
+
+            bool shouldRebuildImage = matchingPlatform.Value.Platform.BaseImageDigest != currentBaseImageDigestReference;
 
             _logger.LogInformation(
-                $"Checking base image '{fromImage}' from '{platform.DockerfilePath}'{Environment.NewLine}"
-                + $"\tLast build digest:    {matchingPlatform.Value.Platform.BaseImageDigest}{Environment.NewLine}"
-                + $"\tCurrent digest:       {currentDigest}{Environment.NewLine}"
-                + $"\tImage is up-to-date:  {!rebuildImage}{Environment.NewLine}");
-
-            if (rebuildImage)
+                "Dockerfile {DockerfilePath} was last built with base image {BaseImagePublicReference} at digest"
+                    + " {LastBuildBaseImageDigestReference}. Image {BaseImagePullReference} has current digest"
+                    + " {CurrentBaseImageDigestReference}. Up to date: {IsUpToDate}.",
+                platform.DockerfilePath,
+                baseImagePublicReference,
+                matchingPlatform.Value.Platform.BaseImageDigest,
+                baseImagePullReference,
+                currentBaseImageDigestReference,
+                !shouldRebuildImage);
+
+            if (shouldRebuildImage)
             {
                 IEnumerable<PlatformInfo> dependentPlatforms = GetDescendants(platform, manifest);
                 return dependentPlatforms.Select(p => p.Model.Dockerfile).ToList();

src/ImageBuilder/Commands/GetStaleImagesOptions.cs

@@ -23,13 +23,30 @@ public class GetStaleImagesOptions : Options, IFilterableOptions, IGitOptionsHos
 
         public BaseImageOverrideOptions BaseImageOverrideOptions { get; set; } = new();
 
+        public string? RegistryOverride { get; set; }
+
+        public string? SourceRepoPrefix { get; set; }
+
         private static readonly GitOptionsBuilder GitBuilder = GitOptionsBuilder.BuildForRepositoryOperations();
 
         private static readonly Argument<string> VariableNameArgument = new(nameof(VariableName))
         {
             Description = "The Azure Pipeline variable name to assign the image paths to"
         };
 
+        private static readonly Option<string?> RegistryOverrideOption = new($"--{ManifestOptions.RegistryOverrideName}")
+        {
+            Description = "Alternative registry that overrides the registry defined in each subscription's manifest. " +
+                "Used together with --source-repo-prefix to redirect external base image lookups to a mirror location."
+        };
+
+        private static readonly Option<string?> SourceRepoPrefixOption = new("--source-repo-prefix")
+        {
+            Description = "Repo prefix used to locate mirrored external base images in the overridden registry " +
+                "(e.g. 'mirror/'). Combined with --registry-override, external FROM tags are resolved against " +
+                "'<registry-override>/<source-repo-prefix><original-repo>:<tag>' instead of their public source."
+        };
+
         public override IEnumerable<Option> GetCliOptions() =>
         [
             ..base.GetCliOptions(),
@@ -38,6 +55,8 @@ public override IEnumerable<Option> GetCliOptions() =>
             ..GitBuilder.GetCliOptions(),
             ..CredentialsOptions.GetCliOptions(),
             ..BaseImageOverrideOptions.GetCliOptions(),
+            RegistryOverrideOption,
+            SourceRepoPrefixOption,
         ];
 
         public override IEnumerable<Argument> GetCliArguments() =>
@@ -64,6 +83,8 @@ public override void Bind(ParseResult result)
             GitBuilder.Bind(result, GitOptions);
             CredentialsOptions.Bind(result);
             BaseImageOverrideOptions.Bind(result);
+            RegistryOverride = result.GetValue(RegistryOverrideOption);
+            SourceRepoPrefix = result.GetValue(SourceRepoPrefixOption);
             VariableName = result.GetValue(VariableNameArgument) ?? string.Empty;
         }
     }