diff --git a/.autover/autover.json b/.autover/autover.json index 4620f2f6e..670e0ba97 100644 --- a/.autover/autover.json +++ b/.autover/autover.json @@ -158,6 +158,10 @@ { "Name": "Amazon.Lambda.AppSyncEvents", "Path": "Libraries/src/Amazon.Lambda.AppSyncEvents/Amazon.Lambda.AppSyncEvents.csproj" + }, + { + "Name": "AWS.OCI.Builder", + "Path": "Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj" } ], "UseCommitsForChangelog": false, diff --git a/.autover/changes/1df94c4d-8841-4630-89ca-9324e3bcaf79.json b/.autover/changes/1df94c4d-8841-4630-89ca-9324e3bcaf79.json new file mode 100644 index 000000000..0cdcce71f --- /dev/null +++ b/.autover/changes/1df94c4d-8841-4630-89ca-9324e3bcaf79.json @@ -0,0 +1,11 @@ +{ + "Projects": [ + { + "Name": "AWS.OCI.Builder", + "Type": "Minor", + "ChangelogMessages": [ + "Initial release of AWS library for programmatically building OCI images without a container runtime" + ] + } + ] +} \ No newline at end of file diff --git a/Libraries/AWS.OCI.Builder.slnf b/Libraries/AWS.OCI.Builder.slnf new file mode 100644 index 000000000..12b00af44 --- /dev/null +++ b/Libraries/AWS.OCI.Builder.slnf @@ -0,0 +1,10 @@ +{ + "solution": { + "path": "Libraries.sln", + "projects": [ + "src\\AWS.OCI.Builder\\AWS.OCI.Builder.csproj", + "test\\AWS.OCI.Builder.IntegrationTests\\AWS.OCI.Builder.IntegrationTests.csproj", + "test\\AWS.OCI.Builder.Tests\\AWS.OCI.Builder.Tests.csproj" + ] + } +} \ No newline at end of file diff --git a/Libraries/Libraries.sln b/Libraries/Libraries.sln index 89a88dd60..0d4118486 100644 --- a/Libraries/Libraries.sln +++ b/Libraries/Libraries.sln @@ -175,6 +175,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestDurableServerlessApp", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestDurableServerlessApp.IntegrationTests", "test\TestDurableServerlessApp.IntegrationTests\TestDurableServerlessApp.IntegrationTests.csproj", "{A89D56F9-63CC-4B47-BFED-693E3410993D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.OCI.Builder", "src\AWS.OCI.Builder\AWS.OCI.Builder.csproj", "{BB37A746-2351-9685-2D1F-1571F8F23E06}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.OCI.Builder.Tests", "test\AWS.OCI.Builder.Tests\AWS.OCI.Builder.Tests.csproj", "{7636C459-D2A3-30DB-4964-CAF22E6BA256}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.OCI.Builder.IntegrationTests", "test\AWS.OCI.Builder.IntegrationTests\AWS.OCI.Builder.IntegrationTests.csproj", "{39FD406D-D77B-9593-CF95-B1BE32F4B7F9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1109,6 +1115,42 @@ Global {A89D56F9-63CC-4B47-BFED-693E3410993D}.Release|x64.Build.0 = Release|Any CPU {A89D56F9-63CC-4B47-BFED-693E3410993D}.Release|x86.ActiveCfg = Release|Any CPU {A89D56F9-63CC-4B47-BFED-693E3410993D}.Release|x86.Build.0 = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|x64.ActiveCfg = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|x64.Build.0 = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|x86.ActiveCfg = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|x86.Build.0 = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|Any CPU.Build.0 = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|x64.ActiveCfg = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|x64.Build.0 = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|x86.ActiveCfg = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|x86.Build.0 = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|x64.ActiveCfg = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|x64.Build.0 = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|x86.ActiveCfg = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|x86.Build.0 = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|Any CPU.Build.0 = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|x64.ActiveCfg = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|x64.Build.0 = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|x86.ActiveCfg = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|x86.Build.0 = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|x64.ActiveCfg = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|x64.Build.0 = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|x86.ActiveCfg = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|x86.Build.0 = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|Any CPU.Build.0 = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|x64.ActiveCfg = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|x64.Build.0 = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|x86.ActiveCfg = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1195,6 +1237,9 @@ Global {0460CF9D-B5CC-47C5-9B30-CEA84695AB3B} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} {616E108A-9ED5-4282-B1D4-7FD49BC6DA2E} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} {A89D56F9-63CC-4B47-BFED-693E3410993D} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} + {BB37A746-2351-9685-2D1F-1571F8F23E06} = {AAB54E74-20B1-42ED-BC3D-CE9F7BC7FD12} + {7636C459-D2A3-30DB-4964-CAF22E6BA256} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {503678A4-B8D1-4486-8915-405A3E9CF0EB} diff --git a/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj b/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj new file mode 100644 index 000000000..73d472b8e --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj @@ -0,0 +1,43 @@ + + + + + + $(DefaultPackageTargets) + AWS.OCI.Builder + AWS.OCI.Builder + AWS.OCI.Builder + A .NET library for building OCI container images programmatically — overlay local files as a new layer on a base image and push the result to Amazon ECR, without a Docker daemon. + enable + enable + + + true + true + true + + + 0.5.0 + AWS.OCI.Builder + aws;ecr;oci;container;docker;image;registry + README.md + https://github.com/aws/aws-lambda-dotnet + git + https://github.com/aws/aws-lambda-dotnet + + + + + + + + + + + <_Parameter1>AWS.OCI.Builder.Tests, PublicKey="0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4" + + + + diff --git a/Libraries/src/AWS.OCI.Builder/Descriptor.cs b/Libraries/src/AWS.OCI.Builder/Descriptor.cs new file mode 100644 index 000000000..9d7da4ca2 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Descriptor.cs @@ -0,0 +1,31 @@ +using System.Text.Json.Serialization; + +namespace AWS.OCI.Builder; + +/// +/// An OCI content descriptor: a typed, content-addressable reference to a blob +/// (a layer, a config, or a manifest) by its media type, size, and digest. +/// See the OCI Image Specification "descriptor" definition. +/// +public sealed record Descriptor +{ + /// The media type of the referenced content (for example an image config or a gzipped layer). + [JsonPropertyName("mediaType")] + public required string MediaType { get; init; } + + /// The size, in bytes, of the referenced content. + [JsonPropertyName("size")] + public required long Size { get; init; } + + /// + /// The digest of the referenced content, in "sha256:<hex>" form. For a layer this is the + /// digest of the compressed (gzipped) tarball — the value used to address the blob in the registry. + /// + [JsonPropertyName("digest")] + public required string Digest { get; init; } + + /// Optional alternate download locations for the content. + [JsonPropertyName("urls")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string[]? Urls { get; init; } +} diff --git a/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs new file mode 100644 index 000000000..ec904cd04 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs @@ -0,0 +1,84 @@ +namespace AWS.OCI.Builder; + +/// +/// Options controlling how an image is pushed to Amazon ECR. +/// +/// +/// All properties are optional. The ECR authorization token and target region are not configured here: the +/// token is passed directly to , and the region is taken from the +/// ECR host in the target reference. +/// +public sealed class EcrPushOptions +{ + /// + /// The default value of : 100 MiB (binary megabytes). Chosen so that + /// common base images — whose individual layers run to tens of MiB — are copied entirely in memory, while + /// genuinely large layers spill to disk. + /// + public const long DefaultMaxInMemoryLayerBytes = 100L * 1024 * 1024; + + /// + /// The size threshold, in bytes, that decides how a base-image layer is copied to ECR during a push. A base + /// layer whose size is at or below this value is buffered in memory; one that exceeds it is streamed through a + /// temporary file on disk instead, bounding peak memory for very large base images. Defaults to + /// (100 MiB, binary). + /// + /// + /// + /// This affects only the copy of base-image layers (which are downloaded from the source registry and + /// re-uploaded to ECR). Layers built by this library are always written to a temporary file when they are added + /// (see ) and uploaded from that file during the push, regardless + /// of this setting. + /// + /// + /// Data integrity is preserved on both paths: the in-memory path verifies the downloaded bytes against the + /// layer's digest before upload, and the spill-to-disk path verifies the digest while streaming to disk + /// and re-reads the file from disk to verify it again before upload — so a faulty disk write that + /// corrupts the spilled layer is caught and the push fails rather than committing corrupt bytes to ECR. The + /// temporary file is deleted after the upload completes (and on failure or cancellation). + /// + /// + /// The value is interpreted as binary bytes. Set it very high to always buffer in memory, or to 0 to + /// always spill to disk. A negative value is rejected at push time. + /// + /// + public long MaxInMemoryLayerBytes { get; init; } = DefaultMaxInMemoryLayerBytes; + + /// + /// An optional sink that receives a event as each layer is pushed, letting a + /// caller observe push progress (for example to render a per-layer progress bar). Defaults to + /// , meaning no progress is reported. + /// + /// + /// + /// Each layer reports a event, zero or more + /// events as its bytes are sent, and a terminal + /// event; a layer the registry already has reports a single + /// event with set. Each event + /// carries the layer's stable / , + /// so the contract holds even if layers are later uploaded concurrently — see . + /// + /// + /// Use for the common case: it captures the current + /// and raises callbacks on it, which keeps UI updates on + /// the UI thread. Any custom implementation must be thread-safe, since a future parallel push may report from + /// multiple threads at once. + /// + /// + public IProgress? Progress { get; init; } + + /// + /// The HTTP timeout applied to the registry client used for the push. Because a whole layer blob is sent in a + /// single PATCH, the default 100-second timeout would cancel a + /// large-layer upload on a slow connection even while the caller's cancellation token is still valid. Defaults + /// to (no client-imposed timeout), so an upload is + /// bounded only by the passed to + /// — pass a token with your own deadline to bound it. Set a finite + /// value to impose a per-request timeout instead. + /// + /// + /// A finite value must be positive, or ; any other value + /// (zero or negative) is rejected at push time. + /// + public TimeSpan UploadTimeout { get; init; } = System.Threading.Timeout.InfiniteTimeSpan; +} diff --git a/Libraries/src/AWS.OCI.Builder/FileManifest.cs b/Libraries/src/AWS.OCI.Builder/FileManifest.cs new file mode 100644 index 000000000..62639c5cf --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/FileManifest.cs @@ -0,0 +1,85 @@ +using System.Collections; + +namespace AWS.OCI.Builder; + +/// +/// Describes a set of local files and/or directories to add to an image, each mapped to an absolute +/// destination path inside the container. A single becomes a single image layer. +/// +/// +/// Collection-initializer syntax is supported, for example: +/// +/// var files = new FileManifest +/// { +/// { "/local/app.dll", "/app/app.dll" }, +/// { "/local/config", "/app/config" }, +/// }; +/// +/// +/// File permissions in the resulting layer: when an entry's is not set, +/// the layer builder derives the Unix mode from the source file on Linux/macOS (preserving an executable bit), +/// and falls back to a standard 0644 for files / 0755 for directories on Windows, which has no Unix +/// mode bits to read. To guarantee a file is executable in the container regardless of the build host — the only +/// way to get the executable bit when building on Windows — use or pass an explicit +/// to . +/// +/// +public sealed class FileManifest : IEnumerable +{ + private readonly List _entries = new(); + + /// The entries that make up this manifest. + public IReadOnlyList Entries => _entries; + + /// + /// Adds a local file or directory to be placed at inside the container, with + /// its Unix mode derived from the source (on Linux/macOS) or defaulted (on Windows). + /// + /// An absolute or relative path to a file or directory on the local filesystem. + /// The absolute destination path inside the container (for example /app/app.dll). + public void Add(string localPath, string containerPath) + => _entries.Add(new FileManifestEntry(localPath, containerPath)); + + /// + /// Adds a local file or directory with an explicit Unix (for example + /// (UnixFileMode)0b111_101_101 for 0755). The explicit mode is applied to the mapped file, or to + /// every file produced by a mapped directory, overriding any source-derived or default mode. + /// + /// An absolute or relative path to a file or directory on the local filesystem. + /// The absolute destination path inside the container. + /// The Unix file mode to stamp on the entry. + public void Add(string localPath, string containerPath, UnixFileMode mode) + => _entries.Add(new FileManifestEntry(localPath, containerPath, mode)); + + /// + /// Adds a local file (or directory of files) and marks it executable (0755) in the container. Use this + /// for scripts and binaries so they are runnable regardless of the build host — on Windows the source file has + /// no executable bit to preserve, so this is the only way to produce an executable entry there. + /// + /// An absolute or relative path to a file or directory on the local filesystem. + /// The absolute destination path inside the container. + public void AddExecutable(string localPath, string containerPath) + => _entries.Add(new FileManifestEntry(localPath, containerPath, FileManifestEntry.ExecutableMode)); + + /// + public IEnumerator GetEnumerator() => _entries.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} + +/// A single local-to-container path mapping within a . +/// The source path on the local filesystem. +/// The absolute destination path inside the container. +/// +/// An explicit Unix file mode for the entry, or to derive it from the source (on +/// Linux/macOS) or use the default (on Windows). For a directory mapping, an explicit mode applies to the files +/// it contains. +/// +public readonly record struct FileManifestEntry(string LocalPath, string ContainerPath, UnixFileMode? Mode = null) +{ + /// A standard executable file mode: 0755 (rwxr-xr-x). + public const UnixFileMode ExecutableMode = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute; +} diff --git a/Libraries/src/AWS.OCI.Builder/ImageReference.cs b/Libraries/src/AWS.OCI.Builder/ImageReference.cs new file mode 100644 index 000000000..f17491caf --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/ImageReference.cs @@ -0,0 +1,180 @@ +using System.Text.RegularExpressions; + +namespace AWS.OCI.Builder; + +/// +/// A parsed container image reference: the registry host, repository, and a tag or digest reference. +/// +/// +/// +/// Handles the common reference forms: +/// +/// +/// mcr.microsoft.com/dotnet/aspnet:8.0 — explicit registry host. +/// 123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest — an ECR reference. +/// library/ubuntu:22.04 / ubuntu — Docker Hub shorthand (host defaults to +/// registry-1.docker.io, the library/ prefix is added for single-segment repositories). +/// A @sha256:... digest reference in place of a tag. +/// +/// +/// The distinction between a registry host and the first path segment follows the Docker rule: the first segment +/// is a host only if it contains a . or :, or is exactly localhost. Otherwise the whole +/// string is a Docker Hub repository. +/// +/// +public sealed partial class ImageReference +{ + private const string DockerHubHost = "registry-1.docker.io"; + + private ImageReference(string registry, string repository, string reference, bool referenceIsDigest) + { + Registry = registry; + Repository = repository; + Reference = reference; + ReferenceIsDigest = referenceIsDigest; + } + + /// The registry host, for example mcr.microsoft.com or an ECR endpoint. + public string Registry { get; } + + /// The repository path, for example dotnet/aspnet or myapp. + public string Repository { get; } + + /// The tag (for example 8.0) or digest (for example sha256:...). + public string Reference { get; } + + /// Whether is a digest () rather than a tag. + public bool ReferenceIsDigest { get; } + + /// Whether this reference points at an Amazon ECR registry. + public bool IsEcr => EcrHostRegex().IsMatch(Registry); + + /// + /// The AWS region embedded in an ECR registry host (for example us-west-2), or + /// when the reference is not an ECR reference. + /// + public string? EcrRegion + { + get + { + Match match = EcrHostRegex().Match(Registry); + return match.Success ? match.Groups["region"].Value : null; + } + } + + /// + /// Parses an image reference string. A missing tag defaults to latest. + /// + /// The reference is null, empty, or malformed. + public static ImageReference Parse(string reference) + { + ArgumentException.ThrowIfNullOrWhiteSpace(reference); + + string remainder = reference; + string registry; + + int firstSlash = remainder.IndexOf('/'); + string firstSegment = firstSlash < 0 ? remainder : remainder[..firstSlash]; + + // The first path segment is a registry host only if it looks like one (contains '.' or ':', or is + // "localhost"); otherwise the whole reference is a Docker Hub repository. + bool firstSegmentIsHost = firstSlash > 0 && + (firstSegment.Contains('.', StringComparison.Ordinal) || + firstSegment.Contains(':', StringComparison.Ordinal) || + firstSegment.Equals("localhost", StringComparison.OrdinalIgnoreCase)); + + if (firstSegmentIsHost) + { + registry = firstSegment; + remainder = remainder[(firstSlash + 1)..]; + } + else + { + registry = DockerHubHost; + } + + // Split off the tag/digest. A '@' marks a digest; otherwise the last ':' after the final '/' is the tag. + string repository; + string refValue; + bool isDigest; + + int atIndex = remainder.IndexOf('@'); + if (atIndex >= 0) + { + string namePortion = remainder[..atIndex]; + refValue = remainder[(atIndex + 1)..]; + isDigest = true; + + // A reference may carry BOTH a tag and a digest (e.g. "dotnet/runtime:8.0@sha256:..."). The digest is + // authoritative, so strip a trailing ":tag" from the name portion; otherwise the tag would leak into + // the repository and address the wrong path. Only a ':' AFTER the final '/' is a tag separator (a ':' + // inside an earlier segment would be a malformed host:port that repository validation rejects anyway). + int nameLastSlash = namePortion.LastIndexOf('/'); + int nameTagColon = namePortion.IndexOf(':', nameLastSlash + 1); + repository = nameTagColon >= 0 ? namePortion[..nameTagColon] : namePortion; + } + else + { + int lastSlash = remainder.LastIndexOf('/'); + int tagColon = remainder.IndexOf(':', lastSlash + 1); + if (tagColon >= 0) + { + repository = remainder[..tagColon]; + refValue = remainder[(tagColon + 1)..]; + } + else + { + repository = remainder; + refValue = "latest"; + } + isDigest = false; + } + + if (repository.Length == 0) + { + throw new ArgumentException($"The image reference '{reference}' has no repository.", nameof(reference)); + } + if (refValue.Length == 0) + { + throw new ArgumentException($"The image reference '{reference}' has an empty tag or digest.", nameof(reference)); + } + + // Docker Hub shorthand: a single-segment repository (e.g. "ubuntu") lives under "library/". + if (registry == DockerHubHost && !repository.Contains('/', StringComparison.Ordinal)) + { + repository = "library/" + repository; + } + + // Validate the repository against the OCI/Docker name grammar. Without this, an input such as + // "123456789012.dkr.ecr.us-west-2.amazonaws.com///attacker.example/app" would be accepted and its + // "//attacker.example/app" repository would override the authority of the request URIs the RegistryClient + // builds — causing the caller's credentials (a StaticRegistryCredentialSource answers for every host) to + // be sent to an attacker-controlled host. Rejecting malformed names here closes that vector. + if (!RepositoryRegex().IsMatch(repository)) + { + throw new ArgumentException( + $"The image reference '{reference}' has an invalid repository '{repository}'. A repository is one or " + + "more lowercase path components (letters, digits, and separators '.', '_', '-') separated by single " + + "'/' characters.", + nameof(reference)); + } + + return new ImageReference(registry, repository, refValue, isDigest); + } + + /// Returns the canonical registry/repository:tag (or @digest) form. + public override string ToString() + => ReferenceIsDigest ? $"{Registry}/{Repository}@{Reference}" : $"{Registry}/{Repository}:{Reference}"; + + // Matches an ECR host and captures the region: .dkr.ecr..amazonaws.com + // (also supports the FIPS and China/GovCloud partitions' domains). + [GeneratedRegex(@"^\d{12}\.dkr\.ecr(-fips)?\.(?[a-z0-9-]+)\.amazonaws\.com(\.cn)?$", RegexOptions.IgnoreCase)] + private static partial Regex EcrHostRegex(); + + // The OCI/Docker repository-name grammar: one or more path components separated by single '/' characters, + // each component lowercase alphanumerics with optional internal '.', '_', '__' or '-' separators. This rejects + // leading/trailing/double slashes, empty components, and uppercase — the shapes that could otherwise smuggle an + // alternate authority into a request URI. + [GeneratedRegex(@"^[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*)*$")] + private static partial Regex RepositoryRegex(); +} diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs new file mode 100644 index 000000000..d58153680 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs @@ -0,0 +1,173 @@ +using System.Security.Cryptography; + +namespace AWS.OCI.Builder.Images; + +/// +/// Assembles a derived image from a base image's config and manifest plus one or more overlaid +/// s. It keeps the config's rootfs.diff_ids/history and the manifest's +/// layers in lockstep, applies config customizations (entrypoint, env, etc.), and produces the final +/// config and manifest bytes together with their digests. +/// +/// +/// +/// This type performs only the in-memory config/manifest manipulation — the "overlay a layer" step. It +/// does not fetch the base image or push the result; those are the registry layer's job. A caller fetches the +/// base manifest and config, constructs an from them, overlays layers, then hands the +/// built artifacts to the registry client for upload. +/// +/// +/// The output preserves the base image's schema family: if the base manifest is a Docker V2 manifest, the derived +/// manifest and the layer descriptors use the Docker media types; otherwise OCI media types are used. This avoids +/// producing a manifest whose media types are inconsistent with its config, which some registries reject. +/// +/// +public sealed class ImageBuilder +{ + private readonly ImageConfig _config; + private readonly Manifest _manifest; + private readonly bool _isDockerSchema; + + private ImageBuilder(ImageConfig config, Manifest manifest, bool isDockerSchema) + { + _config = config; + _manifest = manifest; + _isDockerSchema = isDockerSchema; + } + + /// The media type to use for layers overlaid onto this image (matching the base image's schema). + public string LayerMediaType => _isDockerSchema ? MediaTypes.DockerLayerGzip : MediaTypes.OciLayerGzipV1; + + /// The config blob media type for this image's schema. + public string ConfigMediaType => _isDockerSchema ? MediaTypes.DockerImageConfigV1 : MediaTypes.OciImageConfigV1; + + /// The manifest media type for this image's schema. + public string ManifestMediaType => _isDockerSchema ? MediaTypes.DockerManifestV2 : MediaTypes.OciManifestV1; + + /// The config being assembled, exposed for customization (entrypoint, env, labels, ...). + public ImageConfig Config => _config; + + /// + /// Creates a builder from an already-fetched base image's config and manifest. + /// + /// The base image's config blob JSON. + /// The base image's single-platform manifest JSON (not an index). + /// + /// The media type the registry reported for the base manifest. Used to decide whether the derived image uses + /// Docker or OCI media types. When null, OCI is assumed. + /// + /// The config or manifest JSON is missing or invalid. + public static ImageBuilder FromBaseImage(string baseConfigJson, string baseManifestJson, string? baseManifestMediaType) + { + ImageConfig config = ImageConfig.Parse(baseConfigJson); + Manifest manifest = Manifest.Parse(baseManifestJson); + + bool isDocker = IsDockerSchema(baseManifestMediaType, manifest.MediaType); + return new ImageBuilder(config, manifest, isDocker); + } + + /// + /// Overlays a built onto the image: appends its descriptor to the manifest's + /// layers and its diff_id (plus a history entry) to the config. The layer's media type is + /// coerced to the image's schema so the manifest stays internally consistent. + /// + /// A layer produced by . + /// Optional free-text description recorded in the config history for this layer. + public void AddLayer(Layer layer, string? createdBy = null) + { + ArgumentNullException.ThrowIfNull(layer); + + // Keep the layer descriptor's media type aligned with the image schema rather than trusting whatever the + // layer was built with, so a layer built as OCI can still be overlaid onto a Docker-schema base and vice + // versa without producing a mixed-schema manifest. + _manifest.Layers.Add(new Descriptor + { + MediaType = LayerMediaType, + Size = layer.Size, + Digest = layer.Digest, + }); + + _config.AddLayer(layer.DiffId, createdBy); + } + + /// + /// Finalizes the image: serializes the (mutated) config, points the manifest's config descriptor at the + /// fresh config digest, sets the manifest media type to the image schema, and serializes the manifest. The + /// returned carries everything the registry layer needs to upload the image. + /// + public BuiltImage Build() + { + // 1. Serialize the config and compute its digest; the manifest must reference these exact bytes. + byte[] configBytes = _config.ToJsonBytes(); + string configDigest = ComputeDigest(configBytes); + + _manifest.Config = new Descriptor + { + MediaType = ConfigMediaType, + Size = configBytes.LongLength, + Digest = configDigest, + }; + _manifest.MediaType = ManifestMediaType; + + // 2. Serialize the manifest and compute the image digest over those bytes. + byte[] manifestBytes = _manifest.ToJsonBytes(); + string manifestDigest = ComputeDigest(manifestBytes); + + return new BuiltImage( + configBytes, + configDigest, + ConfigMediaType, + manifestBytes, + manifestDigest, + ManifestMediaType); + } + + /// + /// Decides whether the image should be treated as Docker-schema (vs OCI). The reported manifest media type is + /// the strongest signal; the manifest's own mediaType is next; the config's media type field is a last + /// resort. Anything not clearly Docker defaults to OCI. + /// + private static bool IsDockerSchema(string? reportedManifestMediaType, string? manifestMediaType) + { + foreach (string? candidate in new[] { reportedManifestMediaType, manifestMediaType }) + { + if (string.IsNullOrEmpty(candidate)) + { + continue; + } + if (candidate.StartsWith("application/vnd.docker.", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + if (candidate.StartsWith("application/vnd.oci.", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + } + return false; + } + + private static string ComputeDigest(byte[] data) + { + byte[] hash = SHA256.HashData(data); + // Convert.ToHexStringLower exists only on net9.0+, so lower-case explicitly for net8.0 compatibility. + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } +} + +/// +/// The finished image artifacts produced by : the config and manifest bytes with +/// their digests and media types, ready for the registry layer to upload. +/// +/// The canonical config blob bytes to upload. +/// The sha256:... digest of . +/// The config blob media type. +/// The canonical manifest bytes to upload (and PUT under the target tag). +/// The sha256:... digest of (the image digest). +/// The manifest media type to send as the Content-Type on upload. +public sealed record BuiltImage( + byte[] ConfigBytes, + string ConfigDigest, + string ConfigMediaType, + byte[] ManifestBytes, + string ManifestDigest, + string ManifestMediaType); diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs new file mode 100644 index 000000000..e399041b5 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs @@ -0,0 +1,239 @@ +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace AWS.OCI.Builder.Images; + +/// +/// A mutable view over an OCI / Docker image configuration JSON document — the object referenced by an image +/// manifest's config descriptor. It carries the execution parameters (entrypoint, env, working directory, +/// user, exposed ports, labels) and the rootfs.diff_ids / history that describe the layer stack. +/// +/// +/// +/// The image config schema has many fields and registries reject an image whose config digest does not match its +/// bytes, so this type is deliberately a thin wrapper over a of the original +/// base-image config rather than a hand-modeled POCO. Preserving every field the base set (even ones this library +/// does not understand) keeps the derived image valid and avoids silently dropping data. Mutations touch only the +/// specific nodes they need. +/// +/// +/// Call to get the canonical bytes to upload; the config digest is the SHA-256 of those +/// bytes. All serialization is compact (no insignificant whitespace) and preserves insertion order so the output +/// is stable. +/// +/// +public sealed class ImageConfig +{ + // Writer options for the canonical config bytes: compact (no insignificant whitespace) and relaxed escaping, + // matching what registries emit. The config digest is computed over these exact bytes, so this formatting is + // part of the contract. Writing the JsonNode tree through a Utf8JsonWriter keeps serialization trim/AOT-safe + // (no reflection-based JsonSerializer overload or DefaultJsonTypeInfoResolver). + private static readonly JsonWriterOptions WriterOptions = new() + { + Indented = false, + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + private readonly JsonObject _root; + + private ImageConfig(JsonObject root) => _root = root; + + /// Parses an existing image config document (typically the base image's config blob). + /// The config JSON. + /// The JSON is null, empty, or not a JSON object. + public static ImageConfig Parse(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + throw new ArgumentException("The image config JSON must not be null or empty.", nameof(json)); + } + + JsonNode? node; + try + { + node = JsonNode.Parse(json); + } + catch (JsonException ex) + { + throw new ArgumentException("The image config is not valid JSON.", nameof(json), ex); + } + + if (node is not JsonObject obj) + { + throw new ArgumentException("The image config JSON must be a JSON object.", nameof(json)); + } + + return new ImageConfig(obj); + } + + /// The OS this image targets (the config's os field), for example linux. + public string? Os => _root["os"]?.GetValue(); + + /// The CPU architecture this image targets (the config's architecture field), for example amd64. + public string? Architecture => _root["architecture"]?.GetValue(); + + /// + /// Appends a new layer to the config: adds its to rootfs.diff_ids and records + /// a matching history entry. This must be called for each layer added to the manifest, in the same order. + /// + /// The layer's uncompressed-tar digest (). + /// An optional free-text description recorded in the history entry. + public void AddLayer(string diffId, string? createdBy = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(diffId); + + // rootfs.diff_ids — create the structure if the base config somehow lacked it. + if (_root["rootfs"] is not JsonObject rootfs) + { + rootfs = new JsonObject { ["type"] = "layers" }; + _root["rootfs"] = rootfs; + } + if (rootfs["diff_ids"] is not JsonArray diffIds) + { + diffIds = new JsonArray(); + rootfs["diff_ids"] = diffIds; + } + // Add the diff_id as a JsonNode (not the generic Add, which is trim/AOT-unsafe). + diffIds.Add(JsonValue.Create(diffId) as JsonNode); + + // history — one entry per layer keeps the array aligned with the layer stack. The timestamp is fixed. + if (_root["history"] is not JsonArray history) + { + history = new JsonArray(); + _root["history"] = history; + } + var entry = new JsonObject + { + ["created"] = "1970-01-01T00:00:00Z", + }; + if (!string.IsNullOrEmpty(createdBy)) + { + entry["created_by"] = createdBy; + } + history.Add(entry as JsonNode); + } + + /// Sets the container entrypoint (the config's config.Entrypoint). + public void SetEntrypoint(params string[] entrypoint) + => SetConfigArray("Entrypoint", entrypoint); + + /// Sets the container default command/arguments (the config's config.Cmd). + public void SetCmd(params string[] cmd) + => SetConfigArray("Cmd", cmd); + + /// Sets the working directory (the config's config.WorkingDir). + public void SetWorkingDirectory(string workingDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(workingDirectory); + ConfigObject()["WorkingDir"] = workingDirectory; + } + + /// Sets the user the container runs as (the config's config.User). + public void SetUser(string user) + { + ArgumentException.ThrowIfNullOrWhiteSpace(user); + ConfigObject()["User"] = user; + } + + /// + /// Adds or overwrites an environment variable (merged into the config's config.Env, which is an array of + /// KEY=value strings). An existing entry for the same key is replaced in place. + /// + public void SetEnvironmentVariable(string name, string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(value); + + JsonObject config = ConfigObject(); + if (config["Env"] is not JsonArray env) + { + env = new JsonArray(); + config["Env"] = env; + } + + string prefix = name + "="; + string entry = prefix + value; + for (int i = 0; i < env.Count; i++) + { + if (env[i]?.GetValue() is { } existing && existing.StartsWith(prefix, StringComparison.Ordinal)) + { + env[i] = entry; + return; + } + } + env.Add(JsonValue.Create(entry) as JsonNode); + } + + /// Adds or overwrites a label (the config's config.Labels map). + public void SetLabel(string key, string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(value); + + JsonObject config = ConfigObject(); + if (config["Labels"] is not JsonObject labels) + { + labels = new JsonObject(); + config["Labels"] = labels; + } + labels[key] = value; + } + + /// Marks a port as exposed (added to the config's config.ExposedPorts set). + /// A port spec such as 8080/tcp; a bare number is treated as /tcp. + public void ExposePort(string port) + { + ArgumentException.ThrowIfNullOrWhiteSpace(port); + string key = port.Contains('/', StringComparison.Ordinal) ? port : port + "/tcp"; + + JsonObject config = ConfigObject(); + if (config["ExposedPorts"] is not JsonObject ports) + { + ports = new JsonObject(); + config["ExposedPorts"] = ports; + } + // The value is an empty object per the image-spec convention. + ports[key] = new JsonObject(); + } + + /// Serializes the config to its canonical, compact UTF-8 bytes (what the config digest is computed over). + public byte[] ToJsonBytes() + { + var buffer = new System.Buffers.ArrayBufferWriter(); + using (var writer = new Utf8JsonWriter(buffer, WriterOptions)) + { + _root.WriteTo(writer); + } + return buffer.WrittenSpan.ToArray(); + } + + /// Serializes the config to its canonical, compact JSON string. + public string ToJsonString() => System.Text.Encoding.UTF8.GetString(ToJsonBytes()); + + /// Returns the config sub-object, creating it if absent. + private JsonObject ConfigObject() + { + if (_root["config"] is not JsonObject config) + { + config = new JsonObject(); + _root["config"] = config; + } + return config; + } + + private void SetConfigArray(string key, string[] values) + { + ArgumentNullException.ThrowIfNull(values); + if (values.Length == 0) + { + throw new ArgumentException($"'{key}' must contain at least one element.", nameof(values)); + } + + var array = new JsonArray(); + foreach (string value in values) + { + array.Add(JsonValue.Create(value) as JsonNode); + } + ConfigObject()[key] = array; + } +} diff --git a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs new file mode 100644 index 000000000..47c77420c --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs @@ -0,0 +1,428 @@ +using System.Formats.Tar; +using System.IO.Compression; +using System.Security.Cryptography; + +namespace AWS.OCI.Builder.Images; + +/// +/// The metadata of a single image layer produced from a — a gzipped tar archive of +/// the mapped files — together with the two digests the OCI format requires. The compressed blob itself is +/// written to a caller-supplied stream by ; this type does not hold the layer bytes. +/// +/// +/// +/// An image layer is identified by two different SHA-256 hashes, and the distinction matters: +/// +/// +/// +/// +/// — the hash of the compressed (gzipped) tar bytes. This is the blob's +/// content address: it is what the registry stores the layer under and what the image manifest references. +/// +/// +/// +/// +/// — the hash of the uncompressed tar bytes. This is what the image config's +/// rootfs.diff_ids array references, so the runtime can verify the extracted filesystem. +/// +/// +/// +/// +/// Entries are written in a stable sorted order with fixed timestamps and normalized paths. Digests are not +/// guaranteed to be identical across builds, even for identical input. +/// +/// +public sealed class Layer +{ + /// A fixed timestamp used for every tar entry. + private static readonly DateTimeOffset ReproducibleTimestamp = DateTimeOffset.UnixEpoch; + + private Layer(string digest, string diffId, long size, string mediaType) + { + Digest = digest; + DiffId = diffId; + Size = size; + MediaType = mediaType; + } + + /// + /// The sha256:... digest of the compressed tar. Addresses the blob in the registry and is + /// referenced by the image manifest's layer descriptor. + /// + public string Digest { get; } + + /// + /// The sha256:... digest of the uncompressed tar. Referenced by the image config's + /// rootfs.diff_ids. + /// + public string DiffId { get; } + + /// The size, in bytes, of the compressed tar written to the destination stream. + public long Size { get; } + + /// The layer media type (OCI or Docker gzipped-layer type, matching the target image's schema). + public string MediaType { get; } + + /// + /// A content descriptor for this layer, suitable for inclusion in an image manifest's layers array. + /// + public Descriptor ToDescriptor() => new() + { + MediaType = MediaType, + Size = Size, + Digest = Digest, + }; + + /// + /// Builds a gzipped-tar layer from the files and directories described by , + /// writing the compressed blob to . + /// + /// + /// + /// The layer is produced in a single streaming pass — tar entries are written into a gzip compressor whose + /// output goes straight to — while both digests are computed on the fly: the + /// over the uncompressed tar bytes, and the and + /// over the compressed bytes. Memory use is therefore bounded by stream buffers regardless of layer size, and + /// there is no limit on layer size other than the capacity of the destination. + /// + /// + /// is not disposed, closed, or rewound; the caller owns it. Its prior contents + /// and position are left alone, and the blob is appended starting at its current position. If this method + /// throws, the destination may hold a partial blob and must be discarded. + /// + /// + /// The local-to-container path mappings that populate the layer. + /// A writable stream that receives the compressed (gzipped tar) layer blob. + /// + /// The gzipped-layer media type to report (defaults to ). Pass + /// when overlaying onto a Docker-schema base image. + /// + /// A cancellation token. + /// The built 's metadata, with both digests and the compressed size. + /// or is . + /// + /// The manifest is empty, an entry has an invalid container path, or is not writable. + /// + /// A mapped local file does not exist. + /// A mapped local directory does not exist. + public static async Task CreateAsync( + FileManifest manifest, + Stream destination, + string mediaType = MediaTypes.OciLayerGzipV1, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(manifest); + ArgumentNullException.ThrowIfNull(destination); + if (!destination.CanWrite) + { + throw new ArgumentException("The destination stream must be writable.", nameof(destination)); + } + if (manifest.Entries.Count == 0) + { + throw new ArgumentException("The file manifest is empty; a layer must contain at least one file.", nameof(manifest)); + } + + // Expand every manifest entry into concrete file additions (container path -> local file + mode), plus the + // set of directory paths implied by those files. Using a sorted dictionary keyed by the normalized + // container path gives us a deterministic emission order regardless of manifest insertion order. + var files = new SortedDictionary(StringComparer.Ordinal); + var directories = new SortedSet(StringComparer.Ordinal); + + foreach (FileManifestEntry entry in manifest.Entries) + { + CollectEntry(entry, files, directories); + } + + // One streaming pass: tar -> [hash: diff_id] -> gzip -> [normalize gzip header] -> [hash + count: digest, + // size] -> destination. Nothing is buffered beyond stream buffers, so layer size is not limited by memory + // (an in-memory MemoryStream/byte[] would cap a layer at ~2 GiB and hold several full copies at once). + using var diffIdHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + using var digestHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + + var compressedSink = new HashingWriteStream(destination, digestHash); + var headerNormalizer = new GzipHeaderNormalizingStream(compressedSink); + + // leaveOpen: true — disposing the compressor writes the gzip trailer through the wrappers but must not + // close the caller's destination. SmallestSize keeps layer blobs compact (same level as before). + await using (var gzip = new GZipStream(headerNormalizer, CompressionLevel.SmallestSize, leaveOpen: true)) + { + var tarSink = new HashingWriteStream(gzip, diffIdHash); + await WriteTarAsync(files, directories, tarSink, cancellationToken).ConfigureAwait(false); + } + + await compressedSink.FlushAsync(cancellationToken).ConfigureAwait(false); + + string diffId = ToDigestString(diffIdHash); + string digest = ToDigestString(digestHash); + return new Layer(digest, diffId, compressedSink.BytesWritten, mediaType); + } + + private static string ToDigestString(IncrementalHash hash) + // Convert.ToHexStringLower exists only on net9.0+, so lower-case explicitly for net8.0 compatibility. + => "sha256:" + Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + + /// + /// Expands a single manifest entry into file additions and the directory entries to emit. + /// + /// + /// Only EXPLICITLY mapped directories are recorded in ; the implicit ancestors + /// of a file are not. Tar extraction creates missing parent directories implicitly, so emitting ancestor + /// entries is unnecessary — and harmful, because a synthesized 0755 ancestor entry would overlay (and reset) + /// the base image's own permissions on that directory (for example /tmp's 1777, or a private 0700 + /// parent). Leaving ancestors out preserves the base image's directory metadata. + /// + private static void CollectEntry(FileManifestEntry entry, SortedDictionary files, SortedSet directories) + { + string containerPath = NormalizeContainerPath(entry.ContainerPath); + + // Reject a symlink/reparse point at the mapped path itself: following it could pull in content from + // outside the mapping (including host secrets). + RejectIfSymlink(entry.LocalPath, entry.LocalPath); + + if (File.Exists(entry.LocalPath)) + { + AddFile(entry.LocalPath, containerPath, entry.Mode, files); + return; + } + + if (Directory.Exists(entry.LocalPath)) + { + // A directory maps recursively. Record the mapped directory itself so an empty mapping still appears, + // then walk it WITHOUT following symlinks, recording every descendant file AND directory so the full + // structure (including empty subdirectories) is preserved. + directories.Add(containerPath); + + string root = Path.GetFullPath(entry.LocalPath); + CollectDirectoryTree(root, containerPath, entry.Mode, files, directories); + return; + } + + // Neither a file nor a directory: surface a precise error naming the missing source. + throw new FileNotFoundException( + $"The manifest maps '{entry.LocalPath}' to '{entry.ContainerPath}', but no such file or directory exists.", + entry.LocalPath); + } + + /// + /// Recursively collects the files and subdirectories under into the layer, + /// placing each under . Does not follow directory or file symlinks: a reparse + /// point is rejected rather than traversed, so a link cannot copy files from outside the mapped tree into the + /// image. Empty subdirectories are recorded so the directory structure is preserved. + /// + private static void CollectDirectoryTree( + string localRoot, + string containerRoot, + UnixFileMode? explicitMode, + SortedDictionary files, + SortedSet directories) + { + // Enumerate entries at this level without recursing, so we can inspect each for a reparse point before + // descending. TopDirectoryOnly keeps the symlink decision local to each entry. + foreach (string localEntry in Directory.EnumerateFileSystemEntries(localRoot, "*", SearchOption.TopDirectoryOnly)) + { + string relative = Path.GetRelativePath(localRoot, localEntry).Replace('\\', '/'); + // Normalize/validate the expanded target too (not just caller-supplied destinations): a source name + // containing '..' or backslashes must not be able to escape the container root. + string target = NormalizeContainerPath($"{containerRoot}/{relative}"); + + RejectIfSymlink(localEntry, localEntry); + + if (Directory.Exists(localEntry)) + { + directories.Add(target); + CollectDirectoryTree(localEntry, target, explicitMode, files, directories); + } + else if (File.Exists(localEntry)) + { + AddFile(localEntry, target, explicitMode, files); + } + } + } + + /// + /// Throws when is a symbolic link or other reparse point. The layer builder archives + /// entries as regular files/directories; silently following a link would let a link inside a mapped directory + /// pull in files from outside it (an exfiltration vector) and cyclic links would loop. Rejecting is explicit + /// and safe for this library's scope (overlay a known set of local files). + /// + private static void RejectIfSymlink(string path, string reportedPath) + { + FileSystemInfo? info = Directory.Exists(path) ? new DirectoryInfo(path) + : File.Exists(path) ? new FileInfo(path) + : null; + + if (info is not null && info.Attributes.HasFlag(FileAttributes.ReparsePoint)) + { + throw new ArgumentException( + $"The manifest source '{reportedPath}' is a symbolic link or reparse point, which is not supported. " + + "Map the real target path explicitly instead."); + } + } + + private static void AddFile(string localPath, string containerPath, UnixFileMode? explicitMode, SortedDictionary files) + { + if (files.ContainsKey(containerPath)) + { + throw new ArgumentException($"Two manifest entries map to the same container path '{containerPath}'."); + } + files[containerPath] = new FileSource(localPath, ResolveFileMode(localPath, explicitMode)); + } + + /// + /// Determines the Unix mode to stamp on a file entry: + /// + /// an explicit mode from the manifest entry wins; + /// otherwise, on Linux/macOS the source file's own mode is used, so an executable source stays + /// executable (a non-readable source is still forced readable so the layer is usable); + /// otherwise (Windows, which has no Unix mode bits) a standard 0644 is used. + /// + /// + private static UnixFileMode ResolveFileMode(string localPath, UnixFileMode? explicitMode) + { + if (explicitMode is { } mode) + { + return mode; + } + + if (!OperatingSystem.IsWindows()) + { + // On Unix the source carries real mode bits. Preserve them (notably the executable bits), but always + // ensure the owner can read the file so the resulting layer entry is never unreadable. + UnixFileMode sourceMode = File.GetUnixFileMode(localPath); + return sourceMode | UnixFileMode.UserRead; + } + + return DefaultFileMode; + } + + /// + /// Writes the directory and file entries as a PAX tar archive, in a deterministic order, to + /// (the hashing/compression pipeline). Source files are streamed, not buffered. + /// + /// + /// PAX is the format used by container tooling (including Microsoft.NET.Build.Containers) and is what + /// registries and runtimes expect for image layers. Every entry gets a fixed extended-attribute set (see + /// ) and a fixed rather than + /// the current time. + /// + private static async Task WriteTarAsync( + SortedDictionary files, + SortedSet directories, + Stream destination, + CancellationToken cancellationToken) + { + // leaveOpen: true — the destination is the hashing/compression pipeline, finalized by the caller. + await using (var tar = new TarWriter(destination, TarEntryFormat.Pax, leaveOpen: true)) + { + // Directories first, shallowest-to-deepest (the sorted set already yields a stable order), so every + // file's parent exists before the file entry. Both collections are ordinally sorted for determinism. + foreach (string dir in directories) + { + cancellationToken.ThrowIfCancellationRequested(); + var dirEntry = new PaxTarEntry(TarEntryType.Directory, ToTarEntryName(dir, isDirectory: true), ReproducibleExtendedAttributes()) + { + Mode = DefaultDirectoryMode, + ModificationTime = ReproducibleTimestamp, + }; + await tar.WriteEntryAsync(dirEntry, cancellationToken).ConfigureAwait(false); + } + + foreach ((string containerPath, FileSource file) in files) + { + cancellationToken.ThrowIfCancellationRequested(); + var fileEntry = new PaxTarEntry(TarEntryType.RegularFile, ToTarEntryName(containerPath, isDirectory: false), ReproducibleExtendedAttributes()) + { + Mode = file.Mode, + ModificationTime = ReproducibleTimestamp, + }; + + // Open each source file read-only and shared so concurrent readers (e.g. the build) aren't blocked. + await using FileStream source = new(file.LocalPath, FileMode.Open, FileAccess.Read, FileShare.Read); + fileEntry.DataStream = source; + await tar.WriteEntryAsync(fileEntry, cancellationToken).ConfigureAwait(false); + } + } + } + + /// A resolved file to place in the layer: its source path and the Unix mode to stamp on it. + private readonly record struct FileSource(string LocalPath, UnixFileMode Mode); + + /// + /// The PAX extended attributes applied to every entry: fixed mtime, atime and ctime. + /// + /// + /// On .NET 8 the PAX writer otherwise fills atime and ctime with the current wall-clock time; + /// pinning all three to a fixed epoch keeps them out of the archive. + /// + private static Dictionary ReproducibleExtendedAttributes() + { + string epoch = ReproducibleTimestamp.ToUnixTimeSeconds().ToString("D", System.Globalization.CultureInfo.InvariantCulture); + return new Dictionary + { + ["mtime"] = epoch, + ["atime"] = epoch, + ["ctime"] = epoch, + }; + } + + /// + /// Normalizes a caller-supplied container path to an absolute, forward-slash path with no trailing slash + /// and no redundant segments. + /// + private static string NormalizeContainerPath(string containerPath) + { + if (string.IsNullOrWhiteSpace(containerPath)) + { + throw new ArgumentException("A container path must not be null or empty."); + } + + string normalized = containerPath.Replace('\\', '/').Trim(); + if (!normalized.StartsWith('/')) + { + throw new ArgumentException($"Container path '{containerPath}' must be absolute (start with '/')."); + } + + // Collapse '.' and empty segments; reject '..' which could escape the intended location. + var segments = new List(); + foreach (string segment in normalized.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + if (segment == ".") + { + continue; + } + if (segment == "..") + { + throw new ArgumentException($"Container path '{containerPath}' must not contain '..' segments."); + } + segments.Add(segment); + } + + if (segments.Count == 0) + { + throw new ArgumentException($"Container path '{containerPath}' does not name a file or directory."); + } + + return "/" + string.Join('/', segments); + } + + /// + /// Converts a normalized, rooted container path (for example /app/app.dll) to a tar entry name. Tar + /// entry names are relative (no leading slash) and directory entries carry a trailing slash by convention. + /// + private static string ToTarEntryName(string normalizedPath, bool isDirectory) + { + string name = normalizedPath.TrimStart('/'); + return isDirectory ? name + "/" : name; + } + + // Standard rootfs permissions: 0755 for directories, 0644 for files. + private const UnixFileMode DefaultDirectoryMode = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute; + + private const UnixFileMode DefaultFileMode = + UnixFileMode.UserRead | UnixFileMode.UserWrite | + UnixFileMode.GroupRead | + UnixFileMode.OtherRead; +} diff --git a/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs b/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs new file mode 100644 index 000000000..aeea6769f --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs @@ -0,0 +1,166 @@ +using System.Security.Cryptography; + +namespace AWS.OCI.Builder.Images; + +/// +/// A write-only pass-through stream that feeds every byte written to an and counts +/// them before forwarding to an inner stream. Used to compute a layer's digests (and its compressed size) in the +/// same single pass that writes the layer, so the layer never has to be held in memory. +/// +/// The inner stream is never disposed by this wrapper; its owner remains responsible for it. +internal sealed class HashingWriteStream : Stream +{ + private readonly Stream _inner; + private readonly IncrementalHash _hash; + + public HashingWriteStream(Stream inner, IncrementalHash hash) + { + _inner = inner; + _hash = hash; + } + + /// The total number of bytes written through this stream. + public long BytesWritten { get; private set; } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + + public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count)); + + public override void Write(ReadOnlySpan buffer) + { + _hash.AppendData(buffer); + BytesWritten += buffer.Length; + _inner.Write(buffer); + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + _hash.AppendData(buffer.Span); + BytesWritten += buffer.Length; + return _inner.WriteAsync(buffer, cancellationToken); + } + + public override void Flush() => _inner.Flush(); + public override Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); +} + +/// +/// A write-only pass-through stream that normalizes the 10-byte gzip member header as it is written, so the +/// header carries no build time or OS byte. +/// +/// +/// +/// A gzip member begins with a 10-byte header whose bytes 4–7 hold a modification time (MTIME) and whose byte 9 +/// holds the source OS. fills these with values that vary by run +/// and by runtime (net8.0 writes the current time into MTIME; net9.0+ zeroes it). This stream rewrites MTIME to +/// zero and the OS byte to 0xFF ("unknown") as those bytes pass through. +/// +/// +/// The rewrite is driven by the absolute stream position, so it is correct however the writer chunks the header +/// (one byte at a time, or one write spanning the header and beyond). The caller's buffer is never modified: the +/// header slice is copied before it is patched. The inner stream is never disposed by this wrapper. +/// +/// +internal sealed class GzipHeaderNormalizingStream : Stream +{ + /// Length of the fixed gzip member header (RFC 1952 §2.3). + internal const int HeaderLength = 10; + + private readonly Stream _inner; + private long _position; + + public GzipHeaderNormalizingStream(Stream inner) => _inner = inner; + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + + public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count)); + + public override void Write(ReadOnlySpan buffer) + { + int headerBytes = HeaderBytesIn(buffer.Length); + if (headerBytes > 0) + { + Span patched = stackalloc byte[HeaderLength]; + patched = patched[..headerBytes]; + buffer[..headerBytes].CopyTo(patched); + Normalize(patched, _position); + _inner.Write(patched); + _position += headerBytes; + buffer = buffer[headerBytes..]; + } + + if (!buffer.IsEmpty) + { + _inner.Write(buffer); + _position += buffer.Length; + } + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + int headerBytes = HeaderBytesIn(buffer.Length); + if (headerBytes > 0) + { + byte[] patched = buffer[..headerBytes].ToArray(); + Normalize(patched, _position); + await _inner.WriteAsync(patched, cancellationToken).ConfigureAwait(false); + _position += headerBytes; + buffer = buffer[headerBytes..]; + } + + if (!buffer.IsEmpty) + { + await _inner.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + _position += buffer.Length; + } + } + + public override void Flush() => _inner.Flush(); + public override Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + + /// How many of the next bytes still fall inside the gzip header. + private int HeaderBytesIn(int count) + => _position >= HeaderLength ? 0 : (int)Math.Min(count, HeaderLength - _position); + + /// + /// Patches the header bytes in , which begins at absolute header offset + /// : offsets 4–7 (MTIME) become zero and offset 9 (OS) becomes 0xFF. + /// + private static void Normalize(Span segment, long start) + { + for (int i = 0; i < segment.Length; i++) + { + long headerOffset = start + i; + if (headerOffset is >= 4 and <= 7) + { + segment[i] = 0; + } + else if (headerOffset == 9) + { + segment[i] = 0xFF; + } + } + } +} diff --git a/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs new file mode 100644 index 000000000..06ee4a48a --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs @@ -0,0 +1,157 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AWS.OCI.Builder.Images; + +/// +/// An image manifest (OCI image manifest or Docker V2 Schema 2): the single-platform document that references the +/// image blob and the ordered set of that make up the image. +/// +/// +/// The manifest is serialized compactly and its property order is fixed (schema version, media type, config, +/// layers), because the image's digest/tag is computed over these exact bytes. +/// +public sealed class Manifest +{ + /// The manifest schema version. Always 2 for OCI and Docker V2 Schema 2. + [JsonPropertyName("schemaVersion")] + [JsonPropertyOrder(0)] + public int SchemaVersion { get; set; } = 2; + + /// + /// The manifest media type (for example or + /// ). Some older Docker manifests omit this; it is written when set. + /// + [JsonPropertyName("mediaType")] + [JsonPropertyOrder(1)] + public string? MediaType { get; set; } + + /// The descriptor of the image config blob. + [JsonPropertyName("config")] + [JsonPropertyOrder(2)] + public Descriptor Config { get; set; } = null!; + + /// The ordered layer descriptors (base layers first, then any overlaid layers). + [JsonPropertyName("layers")] + [JsonPropertyOrder(3)] + public List Layers { get; set; } = new(); + + /// Parses an image manifest document. + /// The JSON is null/empty or cannot be parsed as a manifest. + public static Manifest Parse(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + throw new ArgumentException("The manifest JSON must not be null or empty.", nameof(json)); + } + + try + { + return JsonSerializer.Deserialize(json, OciJsonContext.ManifestInfo) + ?? throw new ArgumentException("The manifest JSON deserialized to null.", nameof(json)); + } + catch (JsonException ex) + { + throw new ArgumentException("The manifest is not valid JSON.", nameof(json), ex); + } + } + + /// Serializes the manifest to its canonical, compact UTF-8 bytes (what the image digest is computed over). + public byte[] ToJsonBytes() => JsonSerializer.SerializeToUtf8Bytes(this, OciJsonContext.ManifestInfo); + + /// Serializes the manifest to its canonical, compact JSON string. + public string ToJsonString() => JsonSerializer.Serialize(this, OciJsonContext.ManifestInfo); +} + +/// +/// An image index / manifest list: a multi-platform document whose each point at a +/// single-platform for a specific OS/architecture. Base images are frequently published as +/// an index, so the builder must select the right per-platform manifest from it. +/// +public sealed class ImageIndex +{ + /// The schema version (always 2). + [JsonPropertyName("schemaVersion")] + public int SchemaVersion { get; set; } = 2; + + /// The index media type (OCI image index or Docker manifest list). + [JsonPropertyName("mediaType")] + public string? MediaType { get; set; } + + /// The per-platform manifest descriptors. + [JsonPropertyName("manifests")] + public List Manifests { get; set; } = new(); + + /// Parses an image index / manifest list document. + /// The JSON is null/empty or cannot be parsed as an index. + public static ImageIndex Parse(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + throw new ArgumentException("The image index JSON must not be null or empty.", nameof(json)); + } + + try + { + return JsonSerializer.Deserialize(json, OciJsonContext.ImageIndexInfo) + ?? throw new ArgumentException("The image index JSON deserialized to null.", nameof(json)); + } + catch (JsonException ex) + { + throw new ArgumentException("The image index is not valid JSON.", nameof(json), ex); + } + } + + /// + /// Selects the manifest descriptor matching and , + /// ignoring descriptors that carry no platform or name an "unknown" platform (attestation entries). + /// + /// The matching descriptor, or when the index has no such platform. + public PlatformManifestDescriptor? FindManifest(string os, string architecture) + { + ArgumentException.ThrowIfNullOrWhiteSpace(os); + ArgumentException.ThrowIfNullOrWhiteSpace(architecture); + + return Manifests.FirstOrDefault(m => + m.Platform is { } p && + string.Equals(p.Os, os, StringComparison.OrdinalIgnoreCase) && + string.Equals(p.Architecture, architecture, StringComparison.OrdinalIgnoreCase)); + } +} + +/// A manifest descriptor within an , carrying the target platform. +public sealed class PlatformManifestDescriptor +{ + /// The media type of the referenced manifest. + [JsonPropertyName("mediaType")] + public string MediaType { get; set; } = null!; + + /// The size, in bytes, of the referenced manifest. + [JsonPropertyName("size")] + public long Size { get; set; } + + /// The sha256:... digest of the referenced manifest. + [JsonPropertyName("digest")] + public string Digest { get; set; } = null!; + + /// The platform this manifest targets, if declared. + [JsonPropertyName("platform")] + public Platform? Platform { get; set; } +} + +/// An OS/architecture platform descriptor within an image index. +public sealed class Platform +{ + /// The operating system, for example linux. + [JsonPropertyName("os")] + public string Os { get; set; } = null!; + + /// The CPU architecture, for example amd64 or arm64. + [JsonPropertyName("architecture")] + public string Architecture { get; set; } = null!; + + /// An optional architecture variant, for example v8 for arm64. + [JsonPropertyName("variant")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Variant { get; set; } +} diff --git a/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs b/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs new file mode 100644 index 000000000..5b9c26c75 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs @@ -0,0 +1,67 @@ +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using AWS.OCI.Builder.Registry; + +namespace AWS.OCI.Builder.Images; + +/// +/// The source-generated for the strongly-typed documents this library +/// serializes or deserializes. Routing System.Text.Json calls through this context (rather than the +/// reflection-based JsonSerializer overloads) is what makes those paths trim- and Native-AOT-safe: the +/// serialization metadata is emitted at compile time, so nothing needs runtime reflection or dynamic code +/// generation. +/// +/// +/// +/// Output is compact and omits null-valued properties. The manifest's digest/tag is computed over these exact +/// bytes, so the formatting is part of the contract. (The document is a free-form +/// JsonObject rather than a modeled type, so it is serialized directly through a Utf8JsonWriter in +/// that class rather than through this context.) +/// +/// +/// The relaxed JSON encoder (which leaves characters such as + unescaped) cannot be set on +/// , so it is applied via — a +/// that chains this context's source-generated resolver and adds the encoder. +/// Serializing through the these options produce stays AOT-safe while matching the +/// byte-for-byte escaping other container tooling emits (so the computed digests agree). +/// +/// +[JsonSourceGenerationOptions( + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(Manifest))] +[JsonSerializable(typeof(ImageIndex))] +[JsonSerializable(typeof(Descriptor))] +[JsonSerializable(typeof(PlatformManifestDescriptor))] +[JsonSerializable(typeof(Platform))] +[JsonSerializable(typeof(TokenResponse))] +internal sealed partial class OciJsonContext : JsonSerializerContext +{ + // Chains the source-generated resolver and adds the relaxed encoder. Built lazily (not in a static field + // initializer) because the generated `Default` singleton is itself lazily initialized: referencing it from + // this class's own static-field initializers races its construction and throws a NullReferenceException in + // the type initializer (notably under Native AOT). Lazy defers the reference until first use, by which + // point `Default` is available. + private static readonly Lazy RelaxedOptions = new(() => + new JsonSerializerOptions(Default!.Options) { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }); + + private static readonly Lazy> ManifestInfoLazy = new(() => + (JsonTypeInfo)RelaxedOptions.Value.GetTypeInfo(typeof(Manifest))); + + private static readonly Lazy> ImageIndexInfoLazy = new(() => + (JsonTypeInfo)RelaxedOptions.Value.GetTypeInfo(typeof(ImageIndex))); + + private static readonly Lazy> TokenResponseInfoLazy = new(() => + (JsonTypeInfo)RelaxedOptions.Value.GetTypeInfo(typeof(TokenResponse))); + + /// Type metadata for with the relaxed (digest-faithful) encoder. + internal static JsonTypeInfo ManifestInfo => ManifestInfoLazy.Value; + + /// Type metadata for with the relaxed (digest-faithful) encoder. + internal static JsonTypeInfo ImageIndexInfo => ImageIndexInfoLazy.Value; + + /// Type metadata for (relaxed encoder; this type is only ever read). + internal static JsonTypeInfo TokenResponseInfo => TokenResponseInfoLazy.Value; +} diff --git a/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs b/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs new file mode 100644 index 000000000..660d67f98 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs @@ -0,0 +1,103 @@ +namespace AWS.OCI.Builder; + +/// +/// The lifecycle phase a layer is in when a event is reported. +/// +/// +/// Every layer — whether uploaded or skipped — terminates on , so a consumer counting +/// finished layers increments on a single phase without branching on whether the layer was skipped. A skipped +/// layer is reported as a single event with set +/// to ; it does not emit a preceding because there is no upload to +/// announce. +/// +public enum LayerPushPhase +{ + /// The layer's upload is about to begin. is 0. + Started, + + /// + /// Bytes of the layer have been sent. rises toward + /// across successive events. + /// + Uploading, + + /// + /// The layer is finished. For an uploaded layer equals + /// ; for a skipped layer (already present in the registry) + /// is and + /// is 0. + /// + Completed, +} + +/// +/// Distinguishes a layer copied from the base image from one built locally by this library during the push. +/// +public enum LayerKind +{ + /// A layer of the base image, copied from the source registry into the target repository. + BaseLayer, + + /// A layer produced by this build (via ). + BuiltLayer, +} + +/// +/// A progress event reported while an image is being pushed to Amazon ECR, describing the state of one layer. +/// Subscribe by setting to an of this type. +/// +/// +/// +/// Each layer has a stable identity — , , +/// and — that is assigned once, before any upload begins, and never changes across the events +/// for that layer. Consumers should therefore key their state (for example a per-layer progress bar) by +/// or , not by the order events arrive. +/// +/// +/// This matters because the push may upload layers concurrently in a future revision of this library. When it +/// does, events for different layers will interleave and a layer may finish before an earlier-indexed one — but +/// because each event carries its layer's fixed index and digest, this contract is unchanged and consumers keyed +/// by index/digest continue to work without modification. +/// +/// +/// Only image layers are reported. The config blob and the manifest PUT are not layers and do not +/// produce progress events, so reflects exactly the number of layers (base + built). +/// +/// +public readonly record struct LayerPushProgress +{ + /// The 1-based position of this layer within the push, in the range 1... + public int LayerIndex { get; init; } + + /// The total number of layers in this push (base layers plus layers built by this library). + public int TotalLayers { get; init; } + + /// The layer's sha256:... digest — its stable identity across all events for this layer. + public string Digest { get; init; } + + /// + /// The number of bytes of this layer uploaded so far. 0 at , rising during + /// , and equal to at + /// for an uploaded layer (0 for a skipped one). + /// + public long BytesPushed { get; init; } + + /// + /// The total size of this layer, in bytes. Reported even for a skipped layer (so a UI can show "skipped — + /// 37 MB already present" rather than "0 of 0 bytes"). + /// + public long LayerSizeBytes { get; init; } + + /// + /// when the layer was not uploaded because the registry already has the blob. Such a + /// layer is reported as a single event with + /// of 0. + /// + public bool IsSkipped { get; init; } + + /// The lifecycle phase this event represents. + public LayerPushPhase Phase { get; init; } + + /// Whether this layer is a base-image layer or a layer built by this library. + public LayerKind Kind { get; init; } +} diff --git a/Libraries/src/AWS.OCI.Builder/MediaTypes.cs b/Libraries/src/AWS.OCI.Builder/MediaTypes.cs new file mode 100644 index 000000000..5e5fba11e --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/MediaTypes.cs @@ -0,0 +1,37 @@ +namespace AWS.OCI.Builder; + +/// +/// Well-known media type strings used by the OCI Image Specification and the legacy +/// Docker Image Manifest V2 Schema 2. A base image may use either family; the builder +/// preserves the base image's family when producing the output manifest. +/// +public static class MediaTypes +{ + // --- OCI (application/vnd.oci.*) --- + + /// OCI image manifest (a single-platform image). + public const string OciManifestV1 = "application/vnd.oci.image.manifest.v1+json"; + + /// OCI image index (a multi-platform manifest list). + public const string OciImageIndexV1 = "application/vnd.oci.image.index.v1+json"; + + /// OCI image config. + public const string OciImageConfigV1 = "application/vnd.oci.image.config.v1+json"; + + /// OCI gzipped layer. + public const string OciLayerGzipV1 = "application/vnd.oci.image.layer.v1.tar+gzip"; + + // --- Docker Image Manifest V2 Schema 2 (application/vnd.docker.*) --- + + /// Docker image manifest (a single-platform image). + public const string DockerManifestV2 = "application/vnd.docker.distribution.manifest.v2+json"; + + /// Docker manifest list (a multi-platform manifest list). + public const string DockerManifestListV2 = "application/vnd.docker.distribution.manifest.list.v2+json"; + + /// Docker image config. + public const string DockerImageConfigV1 = "application/vnd.docker.container.image.v1+json"; + + /// Docker gzipped layer. + public const string DockerLayerGzip = "application/vnd.docker.image.rootfs.diff.tar.gzip"; +} diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs new file mode 100644 index 000000000..54b0f5920 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -0,0 +1,669 @@ +using AWS.OCI.Builder.Images; +using AWS.OCI.Builder.Registry; + +namespace AWS.OCI.Builder; + +/// +/// The programmatic entry point for building an OCI container image from a base image plus local files and +/// pushing it to Amazon ECR — no Docker daemon required. +/// +/// +/// +/// The flow mirrors what the .NET SDK's container tooling does internally, specialized for ECR as the push target: +/// +/// +/// Pull the base image's manifest from its source registry (selecting the platform from a manifest list +/// when the base is multi-platform), then its config blob. Base layer blobs are not downloaded unless +/// they must be copied to ECR. +/// Build one gzipped-tar layer per call, overlaying the local files at +/// their target container paths. +/// Apply any config customizations (entrypoint, env, working directory, ...). +/// Copy the base layers to the target ECR repository, upload the new layer(s) and the rewritten config, +/// then PUT the manifest under the destination tag. +/// +/// +/// Typical use: +/// +/// +/// var builder = await OciImageBuilder.FromBaseImageAsync( +/// "mcr.microsoft.com/dotnet/aspnet:8.0", platform: "linux/amd64"); +/// +/// await builder.AddLayerAsync(new FileManifest { { "bin/Release/net8.0/publish", "/app" } }); +/// builder.Config.SetWorkingDirectory("/app"); +/// builder.Config.SetEntrypoint("dotnet", "/app/MyApp.dll"); +/// +/// // Obtain an ECR authorization token (e.g. via AmazonECRClient.GetAuthorizationTokenAsync) and pass it in. +/// await builder.PushToEcrAsync("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest", authorizationToken); +/// +/// This type is not thread-safe; use one instance per image build. +/// +public sealed class OciImageBuilder : IDisposable +{ + private readonly ImageReference _baseReference; + private readonly RegistryClient _sourceClient; + private readonly ImageBuilder _imageBuilder; + private readonly IReadOnlyList _baseLayers; + private readonly List _pendingLayers = new(); + private bool _disposed; + + /// + /// Factory that produces the registry client used to push to the target. Defaults to an ECR-authenticating + /// client built from the push options; replaced in tests to inject a stubbed transport. The default ignores + /// the credential-source argument because builds the ECR source itself. + /// + private Func? _targetClientFactory; + + /// The HTTP timeout for the push client, resolved from at + /// push time and read by the default target-client factory. + private TimeSpan _uploadTimeout = Timeout.InfiniteTimeSpan; + + /// Creates the push-target registry client, honoring the resolved upload timeout. A test override + /// () takes precedence when set. + private RegistryClient CreateTargetClient(string host, IRegistryCredentialSource credentialSource) + => _targetClientFactory is { } factory + ? factory(host, credentialSource) + : RegistryClient.Create(host, credentialSource, _uploadTimeout); + + private OciImageBuilder( + ImageReference baseReference, + RegistryClient sourceClient, + ImageBuilder imageBuilder, + IReadOnlyList baseLayers) + { + _baseReference = baseReference; + _sourceClient = sourceClient; + _imageBuilder = imageBuilder; + _baseLayers = baseLayers; + } + + /// + /// Test-only factory: builds an instance directly from pre-fetched base data and an injected source client, + /// bypassing the live base-image pull. Combine with to exercise + /// the full push flow against stubbed transports. + /// + internal static OciImageBuilder CreateForTest( + ImageReference baseReference, + RegistryClient sourceClient, + ImageBuilder imageBuilder, + IReadOnlyList baseLayers) + => new(baseReference, sourceClient, imageBuilder, baseLayers); + + /// Test-only: overrides how the push-target registry client is created. + internal void UseTargetClientFactoryForTest(Func factory) + => _targetClientFactory = factory; + + /// + /// Test-only: the temp-file paths this builder spilled its built layers to. Lets tests assert on exactly the + /// files this instance owns instead of scanning the shared temp directory, which parallel tests also write to. + /// + internal IReadOnlyList SpilledLayerPathsForTest => _pendingLayers.Select(p => p.FilePath).ToArray(); + + /// The config of the image being built, exposed for customization (entrypoint, env, labels, ...). + public ImageConfig Config => _imageBuilder.Config; + + /// + /// Fetches a base image and prepares a builder to overlay layers onto it. + /// + /// + /// The base image reference, for example mcr.microsoft.com/dotnet/aspnet:8.0 or + /// ubuntu:22.04. Public registries (anonymous or Bearer-token pull) are supported. + /// + /// + /// The target platform as os/architecture (for example linux/amd64 or linux/arm64), used + /// to select the right manifest when the base image is published as a multi-platform manifest list. Defaults to + /// linux/amd64. + /// + /// A cancellation token. + /// The base reference or platform string is malformed. + /// The base image or the requested platform could not be fetched. + public static async Task FromBaseImageAsync( + string baseImage, + string platform = "linux/amd64", + CancellationToken cancellationToken = default) + { + ImageReference baseReference = ImageReference.Parse(baseImage); + (string os, string architecture) = ParsePlatform(platform); + + // Public base images pull anonymously (the handshake still performs a Bearer-token exchange when the + // registry demands one, e.g. Docker Hub; the token is obtained without credentials). + RegistryClient sourceClient = RegistryClient.Create(baseReference.Registry, new AnonymousRegistryCredentialSource()); + + try + { + // 1. Fetch the base reference. It may be a single-platform manifest or a manifest list / image index. + ManifestResponse top = await sourceClient + .GetManifestAsync(baseReference.Repository, baseReference.Reference, cancellationToken) + .ConfigureAwait(false); + + // Verify the top-level document's integrity. When the caller pinned a digest (repo@sha256:...), that + // digest is authoritative and must match. Otherwise fall back to the registry-reported + // Docker-Content-Digest when present — cheap defense-in-depth against a cache/proxy corrupting the + // manifest in transit. (A tag fetch from a registry that omits the header is left unverified here; + // the per-platform manifest, config, and layers it points at are all verified by digest below.) + string? topExpectedDigest = baseReference.ReferenceIsDigest ? baseReference.Reference : top.Digest; + if (topExpectedDigest is not null) + { + RegistryClient.VerifyDigest("base image manifest", topExpectedDigest, top.RawBytes); + } + + ManifestResponse platformManifest = await ResolvePlatformManifestAsync( + sourceClient, baseReference, top, os, architecture, cancellationToken).ConfigureAwait(false); + + // 2. Fetch the config blob referenced by the single-platform manifest, verifying its digest. + Manifest manifest = Manifest.Parse(platformManifest.Json); + if (manifest.Config is null) + { + throw new RegistryException($"The base manifest for '{baseImage}' has no config descriptor."); + } + + byte[] configBytes = await sourceClient + .GetVerifiedBlobBytesAsync(baseReference.Repository, manifest.Config.Digest, "base image config", manifest.Config.Size, cancellationToken) + .ConfigureAwait(false); + string configJson = System.Text.Encoding.UTF8.GetString(configBytes); + + // Validate the platform against the config we actually fetched. ResolvePlatformManifestAsync selects + // the right entry from a manifest LIST, but a single-platform tag/digest is returned without any + // platform comparison — so requesting linux/arm64 from an amd64-only image would otherwise succeed + // and silently push an amd64 image. The config's os/architecture are authoritative, so check them + // here and fail loudly on a mismatch rather than producing a wrong-platform image. + ImageConfig parsedConfig = ImageConfig.Parse(configJson); + if (parsedConfig.Os is { } configOs && !configOs.Equals(os, StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryException( + $"The base image '{baseImage}' is for OS '{configOs}', but '{os}' was requested. " + + "Pass a matching platform, or a base image/tag that provides the requested platform."); + } + if (parsedConfig.Architecture is { } configArch && !configArch.Equals(architecture, StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryException( + $"The base image '{baseImage}' is for architecture '{configArch}', but '{architecture}' was " + + "requested. Pass a matching platform, or a base image/tag that provides the requested platform."); + } + + // 3. Prepare the in-memory overlay builder, preserving the base image's schema (OCI vs Docker). + ImageBuilder imageBuilder = ImageBuilder.FromBaseImage(configJson, platformManifest.Json, platformManifest.MediaType); + + // Capture the base image's layer descriptors now; PushToEcrAsync copies these to ECR before uploading the + // new layer(s). They are a snapshot of the base manifest, distinct from layers added by this build. + IReadOnlyList baseLayers = manifest.Layers.ToArray(); + + return new OciImageBuilder(baseReference, sourceClient, imageBuilder, baseLayers); + } + catch + { + sourceClient.Dispose(); + throw; + } + } + + /// + /// Adds the files described by as a new image layer overlaid on top of the base + /// (and any previously-added) layers. Each call produces one layer. Call this before . + /// + /// The local-to-container path mappings that populate the layer. + /// Optional free-text description recorded in the image history for this layer. + /// A cancellation token. + /// The manifest is empty or an entry has an invalid container path. + /// A mapped local file does not exist. + public async Task AddLayerAsync(FileManifest files, string? createdBy = null, CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + ArgumentNullException.ThrowIfNull(files); + + // Build the layer straight into an owner-only temp file (see SpillFile) in a single streaming pass, so it + // is never held in memory — layers can exceed the ~2 GiB a byte[]/MemoryStream could hold. The builder + // owns the file and deletes it on Dispose (or immediately, if building fails or is cancelled). The media + // type matches the image schema so the manifest stays consistent. + string layerFile = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"aws-oci-built-layer-{Guid.NewGuid():N}.tmp"); + Layer layer; + try + { + await using (var file = Registry.SpillFile.CreateWrite(layerFile)) + { + layer = await Layer.CreateAsync(files, file, _imageBuilder.LayerMediaType, cancellationToken).ConfigureAwait(false); + } + } + catch + { + TryDeleteFile(layerFile); + throw; + } + + // Only now that the blob is safely on disk do we mutate the image builder (adding the layer's descriptor + // and diff_id) and record the pending layer. Doing this before the spill could, on a failed/cancelled + // write, leave the manifest referencing a layer whose blob never materializes — a later push would then + // reference a missing blob, and a retry could duplicate the layer entry. + _imageBuilder.AddLayer(layer, createdBy); + _pendingLayers.Add(new PendingLayer(layer.Digest, layerFile, layer.Size)); + } + + /// + /// Pushes the assembled image to the given Amazon ECR reference, authenticating with an ECR authorization + /// token the caller has already obtained. The target repository must already exist in ECR (this library does + /// not create repositories). The region is taken from the ECR host in . + /// + /// + /// + /// This method is named for its ECR-specific behavior: it expects to be + /// the token returned by the ECR GetAuthorizationToken API — a base64-encoded AWS:<password> + /// string — which the library decodes into the Basic username/password the registry handshake uses. + /// + /// + /// The library deliberately does not depend on the AWS SDK: obtaining the token (and refreshing it if + /// the push outlives its ~12-hour validity) is the caller's responsibility. A typical caller holds an + /// AmazonECRClient (or AWS Tools for PowerShell) already and passes the token straight through, e.g.: + /// + /// + /// var auth = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); + /// string token = auth.AuthorizationData[0].AuthorizationToken; // base64("AWS:<password>") + /// string digest = await builder.PushToEcrAsync(targetImage, token); + /// + /// + /// + /// The destination ECR reference, for example + /// 123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest. + /// + /// + /// The ECR authorization token — the base64-encoded AWS:<password> string from the ECR + /// GetAuthorizationToken API's AuthorizationData.AuthorizationToken. + /// + /// Optional push settings (for example the in-memory layer threshold). + /// A cancellation token. + /// The pushed image's manifest digest (sha256:...). + /// + /// The target reference is not a valid ECR reference, the authorization token is missing or malformed, or an + /// option value is invalid. + /// + /// A registry operation failed. + public async Task PushToEcrAsync( + string targetImage, + string authorizationToken, + EcrPushOptions? options = null, + CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + + ImageReference target = ImageReference.Parse(targetImage); + if (!target.IsEcr) + { + throw new ArgumentException( + $"The push target '{targetImage}' is not an Amazon ECR reference. This library only pushes to ECR " + + "(e.g. 123456789012.dkr.ecr.us-west-2.amazonaws.com/repo:tag).", + nameof(targetImage)); + } + + long maxInMemoryLayerBytes = options?.MaxInMemoryLayerBytes ?? EcrPushOptions.DefaultMaxInMemoryLayerBytes; + if (maxInMemoryLayerBytes < 0) + { + throw new ArgumentException( + $"{nameof(EcrPushOptions.MaxInMemoryLayerBytes)} must be non-negative (was {maxInMemoryLayerBytes}).", + nameof(options)); + } + + TimeSpan uploadTimeout = options?.UploadTimeout ?? Timeout.InfiniteTimeSpan; + if (uploadTimeout != Timeout.InfiniteTimeSpan && uploadTimeout <= TimeSpan.Zero) + { + throw new ArgumentException( + $"{nameof(EcrPushOptions.UploadTimeout)} must be positive or Timeout.InfiniteTimeSpan (was {uploadTimeout}).", + nameof(options)); + } + _uploadTimeout = uploadTimeout; + + // Decode the ECR authorization token into the Basic username/password the registry handshake expects. + (string username, string password) = DecodeEcrAuthorizationToken(authorizationToken); + var credentialSource = new StaticRegistryCredentialSource(username, password); + + IProgress? progress = options?.Progress; + + using RegistryClient targetClient = CreateTargetClient(target.Registry, credentialSource); + + // 1. Finalize the config + manifest bytes and their digests. + BuiltImage built = _imageBuilder.Build(); + + // Assign each layer its stable identity (index, total, digest, size, kind) BEFORE any upload begins. Base + // layers (in manifest order) come first, then the locally-built layers (in add order). Fixing identity up + // front is what makes the progress contract survive a future parallel push: an event always carries its + // layer's own index/digest, never a position derived from the order work happens to finish in. + int totalLayers = _baseLayers.Count + _pendingLayers.Count; + + // 2. Copy the base image's layers from the source registry to the ECR target. Cross-host, so each base + // layer that ECR does not already have is streamed from the source and uploaded. Base layers take the + // first index slots; the call returns the next free index for the locally-built layers. + int nextIndex = await CopyBaseLayersAsync(target.Repository, targetClient, maxInMemoryLayerBytes, progress, totalLayers, cancellationToken) + .ConfigureAwait(false); + + // 3. Upload the new layer blob(s) produced locally. Each was spilled to a temp file by AddLayerAsync, so + // it is streamed from disk here rather than held in memory. + foreach (PendingLayer pending in _pendingLayers) + { + int layerIndex = nextIndex++; + long size = pending.Size; + + if (await targetClient.BlobExistsAsync(target.Repository, pending.Digest, cancellationToken).ConfigureAwait(false)) + { + ReportSkipped(progress, layerIndex, totalLayers, pending.Digest, size, LayerKind.BuiltLayer); + continue; + } + + ReportStarted(progress, layerIndex, totalLayers, pending.Digest, size, LayerKind.BuiltLayer); + + await using (var layerStream = new FileStream( + pending.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, useAsync: true)) + { + await targetClient.UploadBlobAsync( + target.Repository, + pending.Digest, + layerStream, + cancellationToken, + onBytesSent: sent => ReportUploading(progress, layerIndex, totalLayers, pending.Digest, sent, size, LayerKind.BuiltLayer), + contentLength: size) + .ConfigureAwait(false); + } + + ReportCompleted(progress, layerIndex, totalLayers, pending.Digest, size, LayerKind.BuiltLayer); + } + + // 4. Upload the config blob. + using (var configStream = new MemoryStream(built.ConfigBytes, writable: false)) + { + await targetClient.UploadBlobAsync( + target.Repository, built.ConfigDigest, configStream, cancellationToken, contentLength: built.ConfigBytes.LongLength) + .ConfigureAwait(false); + } + + // 5. PUT the manifest under the destination tag/digest. All referenced blobs are now present. + string manifestJson = System.Text.Encoding.UTF8.GetString(built.ManifestBytes); + await targetClient.PutManifestAsync(target.Repository, target.Reference, manifestJson, built.ManifestMediaType, cancellationToken) + .ConfigureAwait(false); + + return built.ManifestDigest; + } + + /// + /// Copies every base-image layer from the source registry into the target ECR repository, skipping any blob + /// ECR already has. Base layers come from a different host than ECR, so cross-repo mount cannot be used — the + /// blob is streamed from the source and re-uploaded. + /// + /// + /// A layer whose declared size is at or below is buffered in memory; + /// a larger one is streamed through a temporary file on disk to bound peak memory. Both paths verify the + /// layer's digest before upload (the disk path verifies twice — see + /// ), so data integrity holds regardless of which path + /// a layer takes. + /// + private async Task CopyBaseLayersAsync( + string targetRepository, + RegistryClient targetClient, + long maxInMemoryLayerBytes, + IProgress? progress, + int totalLayers, + CancellationToken cancellationToken) + { + // Base layers occupy the first index slots (1..baseCount). The local counter advances per layer and its + // final value — the next free index — is returned for the locally-built layers that follow. + int index = 1; + foreach (Descriptor layer in _baseLayers) + { + int layerIndex = index++; + long size = layer.Size; + + if (await targetClient.BlobExistsAsync(targetRepository, layer.Digest, cancellationToken).ConfigureAwait(false)) + { + ReportSkipped(progress, layerIndex, totalLayers, layer.Digest, size, LayerKind.BaseLayer); + continue; + } + + ReportStarted(progress, layerIndex, totalLayers, layer.Digest, size, LayerKind.BaseLayer); + + if (layer.Size > maxInMemoryLayerBytes) + { + // Large layer: stream to a temp file (verified twice) and upload from disk, bounding memory. + await using VerifiedBlobFile spilled = await _sourceClient + .GetVerifiedBlobToFileAsync(_baseReference.Repository, layer.Digest, "base layer", layer.Size, cancellationToken) + .ConfigureAwait(false); + + await targetClient.UploadBlobAsync( + targetRepository, + layer.Digest, + spilled.Stream, + cancellationToken, + onBytesSent: sent => ReportUploading(progress, layerIndex, totalLayers, layer.Digest, sent, size, LayerKind.BaseLayer), + contentLength: size) + .ConfigureAwait(false); + } + else + { + // Small layer: download, verify, and re-upload from memory. + byte[] blob = await _sourceClient + .GetVerifiedBlobBytesAsync(_baseReference.Repository, layer.Digest, "base layer", layer.Size, cancellationToken) + .ConfigureAwait(false); + + using var upload = new MemoryStream(blob, writable: false); + await targetClient.UploadBlobAsync( + targetRepository, + layer.Digest, + upload, + cancellationToken, + onBytesSent: sent => ReportUploading(progress, layerIndex, totalLayers, layer.Digest, sent, size, LayerKind.BaseLayer), + contentLength: blob.LongLength) + .ConfigureAwait(false); + } + + ReportCompleted(progress, layerIndex, totalLayers, layer.Digest, size, LayerKind.BaseLayer); + } + + return index; + } + + // ---- Progress reporting helpers ------------------------------------------------------------ + // Each helper builds one LayerPushProgress event and hands it to the sink (when one is configured). Pulling + // these out keeps the push methods readable and guarantees every event sets the same stable-identity fields. + + private static void ReportStarted( + IProgress? progress, int index, int total, string digest, long size, LayerKind kind) + => progress?.Report(new LayerPushProgress + { + LayerIndex = index, + TotalLayers = total, + Digest = digest, + BytesPushed = 0, + LayerSizeBytes = size, + IsSkipped = false, + Phase = LayerPushPhase.Started, + Kind = kind, + }); + + private static void ReportUploading( + IProgress? progress, int index, int total, string digest, long bytesPushed, long size, LayerKind kind) + => progress?.Report(new LayerPushProgress + { + LayerIndex = index, + TotalLayers = total, + Digest = digest, + BytesPushed = bytesPushed, + LayerSizeBytes = size, + IsSkipped = false, + Phase = LayerPushPhase.Uploading, + Kind = kind, + }); + + private static void ReportCompleted( + IProgress? progress, int index, int total, string digest, long size, LayerKind kind) + => progress?.Report(new LayerPushProgress + { + LayerIndex = index, + TotalLayers = total, + Digest = digest, + BytesPushed = size, + LayerSizeBytes = size, + IsSkipped = false, + Phase = LayerPushPhase.Completed, + Kind = kind, + }); + + private static void ReportSkipped( + IProgress? progress, int index, int total, string digest, long size, LayerKind kind) + => progress?.Report(new LayerPushProgress + { + LayerIndex = index, + TotalLayers = total, + Digest = digest, + BytesPushed = 0, + LayerSizeBytes = size, + IsSkipped = true, + Phase = LayerPushPhase.Completed, + Kind = kind, + }); + + private static async Task ResolvePlatformManifestAsync( + RegistryClient client, + ImageReference baseReference, + ManifestResponse top, + string os, + string architecture, + CancellationToken cancellationToken) + { + // Single-platform manifest — use it directly. + if (!IsIndexMediaType(top.MediaType)) + { + return top; + } + + // Manifest list / image index — pick the matching platform and fetch that manifest by digest, then verify + // the returned bytes hash to the digest we requested it under. + ImageIndex index = ImageIndex.Parse(top.Json); + PlatformManifestDescriptor? selected = index.FindManifest(os, architecture) + ?? throw new RegistryException( + $"The base image '{baseReference}' has no manifest for platform '{os}/{architecture}'. " + + $"Available: {DescribePlatforms(index)}."); + + ManifestResponse platformManifest = + await client.GetManifestAsync(baseReference.Repository, selected.Digest, cancellationToken).ConfigureAwait(false); + + RegistryClient.VerifyDigest( + $"platform manifest {os}/{architecture}", + selected.Digest, + platformManifest.RawBytes); + + return platformManifest; + } + + private static string DescribePlatforms(ImageIndex index) + { + IEnumerable platforms = index.Manifests + .Where(m => m.Platform is not null && !string.Equals(m.Platform!.Os, "unknown", StringComparison.OrdinalIgnoreCase)) + .Select(m => $"{m.Platform!.Os}/{m.Platform.Architecture}"); + string joined = string.Join(", ", platforms); + return joined.Length == 0 ? "(none)" : joined; + } + + private static bool IsIndexMediaType(string mediaType) + => string.Equals(mediaType, MediaTypes.OciImageIndexV1, StringComparison.OrdinalIgnoreCase) + || string.Equals(mediaType, MediaTypes.DockerManifestListV2, StringComparison.OrdinalIgnoreCase); + + private static (string Os, string Architecture) ParsePlatform(string platform) + { + ArgumentException.ThrowIfNullOrWhiteSpace(platform); + string[] parts = platform.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + throw new ArgumentException( + $"The platform '{platform}' must be in 'os/architecture' form, for example 'linux/amd64'.", + nameof(platform)); + } + return (parts[0], parts[1]); + } + + private void ThrowIfDisposed() + => ObjectDisposedException.ThrowIf(_disposed, this); + + /// + /// Decodes an ECR authorization token (base64 of username:password, where the username is AWS) + /// into its username/password parts. + /// + /// The token is null/empty, not valid base64, or not in the expected form. + private static (string Username, string Password) DecodeEcrAuthorizationToken(string authorizationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(authorizationToken); + + byte[] decodedBytes; + try + { + decodedBytes = Convert.FromBase64String(authorizationToken.Trim()); + } + catch (FormatException ex) + { + throw new ArgumentException( + "The ECR authorization token is not valid base64. Pass the AuthorizationData.AuthorizationToken " + + "value returned by the ECR GetAuthorizationToken API verbatim.", + nameof(authorizationToken), ex); + } + + string decoded = System.Text.Encoding.UTF8.GetString(decodedBytes); + int separator = decoded.IndexOf(':'); + if (separator < 0) + { + throw new ArgumentException( + "The ECR authorization token did not decode to the expected 'username:password' form.", + nameof(authorizationToken)); + } + + string username = decoded[..separator]; + string password = decoded[(separator + 1)..]; + + // Fail fast on a structurally broken token rather than letting it surface later as an opaque registry + // authentication error. The username is not pinned to "AWS" (ECR's documented value) to avoid coupling + // to that detail; an empty username or password, though, can never be a valid ECR credential. + if (username.Length == 0 || password.Length == 0) + { + throw new ArgumentException( + "The ECR authorization token decoded to a 'username:password' pair with an empty " + + (username.Length == 0 ? "username" : "password") + ". Pass the AuthorizationData.AuthorizationToken " + + "value returned by the ECR GetAuthorizationToken API verbatim.", + nameof(authorizationToken)); + } + + return (username, password); + } + + /// Disposes the source registry client and deletes any temp files holding spilled built layers. + public void Dispose() + { + if (_disposed) + { + return; + } + _disposed = true; + + foreach (PendingLayer pending in _pendingLayers) + { + TryDeleteFile(pending.FilePath); + } + _pendingLayers.Clear(); + + _sourceClient.Dispose(); + } + + /// Best-effort delete of a temp file; a leftover must never mask a more important error. + private static void TryDeleteFile(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch + { + // Best-effort cleanup only. + } + } + + /// + /// A layer built locally during this session. The compressed blob is spilled to (not + /// kept in memory) and streamed from there by ; the builder deletes the file on + /// . + /// + private readonly record struct PendingLayer(string Digest, string FilePath, long Size); +} diff --git a/Libraries/src/AWS.OCI.Builder/README.md b/Libraries/src/AWS.OCI.Builder/README.md new file mode 100644 index 000000000..5387828a2 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/README.md @@ -0,0 +1,135 @@ +# AWS.OCI.Builder + +A .NET library for building [OCI](https://opencontainers.org/) container images +**programmatically**, without a Docker daemon. It pulls a base image's manifest over the +registry HTTP API, overlays a new layer containing local files at specified container paths, +and pushes the resulting image to **Amazon ECR**. + +## Status + +This library is designed and supported for use by other AWS libraries and tools. +The version of the library is kept below 1.0.0 to indicate that the API is subject to change based on the needs of AWS libraries and tools. +If you are using this library in a non-AWS context, please be aware that the API may change in future releases. + +## Authentication + +This library has **no AWS SDK dependency**. Instead of resolving AWS credentials itself, it takes an ECR +authorization token that you obtain and pass to `PushToEcrAsync`. This keeps the library free of +`AWSSDK.Core`, so it will not conflict with the AWS SDK versions a host (for example an AWS Tools for +PowerShell cmdlet) already loads. + +The token is the value returned by the ECR +[`GetAuthorizationToken`](https://docs.aws.amazon.com/AmazonECR/latest/APIReference/API_GetAuthorizationToken.html) +API — a base64-encoded `AWS:` string. The library decodes it into the `Basic` username/password the +registry handshake needs. Obtaining and refreshing the token (it is valid for ~12 hours) is the caller's +responsibility; the target region is taken from the ECR host in the target reference. + +```csharp +using Amazon.ECR; +using Amazon.ECR.Model; + +// Caller obtains the token however it likes — directly via the SDK, or from an AWS Tools for PowerShell cmdlet. +using var ecr = new AmazonECRClient(); // region/credentials from the SDK default chain +var auth = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); +string authorizationToken = auth.AuthorizationData[0].AuthorizationToken; // base64("AWS:") +``` + +## Usage + +```csharp +using AWS.OCI.Builder; + +// Pull the base image (selects the right platform from a manifest list). +using var builder = await OciImageBuilder.FromBaseImageAsync( + "mcr.microsoft.com/dotnet/aspnet:8.0", + platform: "linux/amd64"); + +// Overlay local files as a new layer. Each call adds one layer. +await builder.AddLayerAsync(new FileManifest +{ + { "/local/publish/MyApp.dll", "/app/MyApp.dll" }, + { "/local/publish/config", "/app/config" }, +}); + +// Setting Linux file permissions (see "File permissions" below): +var files = new FileManifest(); +files.Add("/local/publish/MyApp.dll", "/app/MyApp.dll"); // default 0644 +files.AddExecutable("/local/scripts/entrypoint.sh", "/app/entrypoint.sh"); // 0755 +files.Add("/local/secrets/key", "/app/key", UnixPermissions.FromOctal("600")); // explicit 0600 +await builder.AddLayerAsync(files); + +// Customize the image config. +builder.Config.SetWorkingDirectory("/app"); +builder.Config.SetEntrypoint("dotnet", "/app/MyApp.dll"); + +// Environment variables are merged into the base image's existing config.Env. Setting a key the +// base image already defines (e.g. PATH) overrides it in place rather than duplicating it. +builder.Config.SetEnvironmentVariable("ASPNETCORE_URLS", "http://+:8080"); +builder.Config.SetEnvironmentVariable("DOTNET_EnableDiagnostics", "0"); + +// Obtain an ECR authorization token (the library has no AWS SDK dependency — getting the token is the +// caller's job; see "Authentication" above), then push. The region is taken from the ECR host in the target. +string digest = await builder.PushToEcrAsync( + "123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest", + authorizationToken); +``` + +### Tracking push progress + +Pass an `IProgress` via `EcrPushOptions.Progress` to observe the push layer by layer — for +example to render a progress bar. It is entirely opt-in; omit it and the push reports nothing. + +```csharp +var progress = new Progress(p => +{ + switch (p.Phase) + { + case LayerPushPhase.Started: + Console.WriteLine($"[{p.LayerIndex}/{p.TotalLayers}] {p.Kind} {p.Digest} — uploading {p.LayerSizeBytes:N0} bytes"); + break; + case LayerPushPhase.Uploading: + Console.WriteLine($"[{p.LayerIndex}/{p.TotalLayers}] {p.BytesPushed:N0} / {p.LayerSizeBytes:N0} bytes"); + break; + case LayerPushPhase.Completed when p.IsSkipped: + Console.WriteLine($"[{p.LayerIndex}/{p.TotalLayers}] {p.Digest} — already present, skipped"); + break; + case LayerPushPhase.Completed: + Console.WriteLine($"[{p.LayerIndex}/{p.TotalLayers}] {p.Digest} — done"); + break; + } +}); + +string digest = await builder.PushToEcrAsync( + "123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest", + authorizationToken, + new EcrPushOptions { Progress = progress }); +``` + +Each layer reports a `Started` event, zero or more `Uploading` events as its bytes are sent, and a terminal +`Completed` event. A layer the registry already has reports a **single** `Completed` event with `IsSkipped = true` +and `BytesPushed = 0` — so "layer done" is always the one `Completed` phase, whether the layer was uploaded or +skipped. Only image layers are reported (the config blob and manifest are not layers), so `TotalLayers` is exactly +the number of layers. + +Every event carries the layer's stable `LayerIndex` (1-based), `TotalLayers`, `Digest` and `Kind` +(`BaseLayer` vs `BuiltLayer`), all fixed before any upload begins. Key your state by `LayerIndex` or `Digest` +rather than the order events arrive: the contract is designed so that if the library later uploads layers in +parallel — interleaving their events — consumers keep working unchanged. A custom `IProgress` implementation +must be thread-safe for that reason; `System.Progress` already marshals callbacks to the captured +synchronization context. + +### File permissions + +Each file in a layer carries a Unix mode. When you don't specify one: + +- On **Linux/macOS** the source file's own mode is preserved, so an already-executable file stays executable. +- On **Windows** — which has no Unix mode bits to read — files default to `0644` and directories to `0755`. + +Because Windows can't carry an executable bit, set the mode explicitly when you need a script or binary to be +runnable in the container regardless of the build host: + +- `FileManifest.AddExecutable(local, container)` — marks the entry `0755`. +- `FileManifest.Add(local, container, mode)` — any `System.IO.UnixFileMode`. +- `UnixPermissions.FromOctal("755")` / `FromOctal(755)` — express a mode in `chmod`-style octal (C# has no octal + literal, so `FromOctal` is how you write `0755`). For a directory mapping, an explicit mode applies to every file + it contains; directory entries themselves are always `0755`. diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs new file mode 100644 index 000000000..09f4e75bd --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -0,0 +1,444 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Web; + +namespace AWS.OCI.Builder.Registry; + +/// +/// A that performs the Docker/OCI registry authentication handshake. +/// +/// +/// +/// When a registry responds 401 Unauthorized with a WWW-Authenticate challenge, this handler: +/// +/// +/// parses the challenge scheme (Bearer or Basic); +/// +/// for Bearer, validates the realm endpoint (see ) and performs a +/// token GET against it, sending the resolved credentials as Basic auth and reading back a token; +/// +/// for Basic, encodes the resolved credentials directly; +/// caches the resulting authorization header per registry host (with expiry for Bearer tokens) and retries the request. +/// +/// +/// Credentials come from an . Anonymous access is attempted when the +/// source returns (public base-image pulls do not need credentials to fetch a token). +/// +/// +public sealed partial class AuthHandshakeHandler : DelegatingHandler +{ + private const string BasicScheme = "Basic"; + private const string BearerScheme = "Bearer"; + + private readonly IRegistryCredentialSource _credentialSource; + + // Cached authorization header per registry host, with the time it stops being valid. + private readonly ConcurrentDictionary _cache = new(StringComparer.OrdinalIgnoreCase); + + /// + /// When set to on a request's , this handler + /// sends the request without attaching cached credentials and without answering a 401 challenge. Used for + /// cross-host blob redirect hops so credentials are never presented to a redirect target. + /// + internal static readonly HttpRequestOptionsKey SkipAuthentication = new("AWS.OCI.Builder.SkipAuthentication"); + + /// Creates the handler. + /// Supplies credentials for each registry host. + /// The inner handler (typically an ). + public AuthHandshakeHandler(IRegistryCredentialSource credentialSource, HttpMessageHandler innerHandler) + : base(innerHandler) + => _credentialSource = credentialSource ?? throw new ArgumentNullException(nameof(credentialSource)); + + private readonly record struct CachedAuth(AuthenticationHeaderValue Header, DateTimeOffset Expiry); + + /// + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri is null) + { + throw new ArgumentException("The request must have an absolute RequestUri.", nameof(request)); + } + + // Requests explicitly marked to skip authentication (e.g. blob-download redirect hops to a different host, + // such as object storage) are sent as-is: no cached header is attached and a 401 challenge is NOT + // answered. Otherwise a malicious registry could redirect a blob GET to a host it controls, reply 401 + // with a challenge, and harvest the credentials this handler would send in response. + if (request.Options.TryGetValue(SkipAuthentication, out bool skip) && skip) + { + return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + } + + // Respect a caller-supplied Authorization header: send the request unchanged and let the caller handle + // any 401, rather than overwriting their credentials with cached or handshake-derived ones. + if (request.Headers.Authorization is not null) + { + return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + } + + string registryHost = request.RequestUri.Host; + + // Apply a cached, still-valid authorization header if we have one. + if (_cache.TryGetValue(registryHost, out CachedAuth cached) && cached.Expiry > DateTimeOffset.UtcNow) + { + request.Headers.Authorization = cached.Header; + } + + HttpResponseMessage response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (response.StatusCode != HttpStatusCode.Unauthorized) + { + return response; + } + + // 401 -> parse the challenge and perform the handshake. + if (!TryParseChallenge(response, out string? scheme, out AuthInfo? authInfo)) + { + return response; // No usable challenge; let the caller handle the 401. + } + + // Drain the 401 body so the connection is freed for the token request. + _ = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + response.Dispose(); + + (AuthenticationHeaderValue header, DateTimeOffset expiry) = + await AuthenticateAsync(registryHost, scheme!, authInfo, cancellationToken).ConfigureAwait(false); + + _cache[registryHost] = new CachedAuth(header, expiry); + + // Retry the original request with the authorization header. The clone is disposed once sent; its content + // never owns the caller's body stream (see CloneAsync), so disposing it cannot close that stream. + using HttpRequestMessage retry = await CloneAsync(request).ConfigureAwait(false); + retry.Headers.Authorization = header; + return await base.SendAsync(retry, cancellationToken).ConfigureAwait(false); + } + + private async Task<(AuthenticationHeaderValue, DateTimeOffset)> AuthenticateAsync( + string registryHost, string scheme, AuthInfo? authInfo, CancellationToken cancellationToken) + { + BasicRegistryCredentials? creds = + await _credentialSource.GetCredentialsAsync(registryHost, cancellationToken).ConfigureAwait(false); + + if (scheme.Equals(BasicScheme, StringComparison.OrdinalIgnoreCase)) + { + if (creds is not { } basic) + { + throw new RegistryAuthenticationException(registryHost, "the registry requires Basic auth but no credentials were available."); + } + return (BasicHeader(basic), DateTimeOffset.MaxValue); + } + + if (scheme.Equals(BearerScheme, StringComparison.OrdinalIgnoreCase)) + { + if (authInfo is not { } info) + { + throw new InvalidAuthResponseException(registryHost, "the Bearer challenge did not include a realm."); + } + return await GetBearerTokenAsync(registryHost, info, creds, cancellationToken).ConfigureAwait(false); + } + + throw new RegistryAuthenticationException(registryHost, $"unsupported authentication scheme '{scheme}'."); + } + + private async Task<(AuthenticationHeaderValue, DateTimeOffset)> GetBearerTokenAsync( + string registryHost, AuthInfo info, BasicRegistryCredentials? creds, CancellationToken cancellationToken) + { + // SSRF guard: never send credentials to an unvalidated realm. + Uri realm = RealmValidator.Validate(info.Realm, registryHost); + + var builder = new UriBuilder(realm); + System.Collections.Specialized.NameValueCollection query = HttpUtility.ParseQueryString(realm.Query); + if (info.Service is { } svc) + { + query["service"] = svc; + } + if (info.Scope is { } scope) + { + query["scope"] = scope; + } + builder.Query = query.ToString(); + + using var tokenRequest = new HttpRequestMessage(HttpMethod.Get, builder.Uri); + if (creds is { } c) + { + // Even for a Bearer scheme, username/password are presented to the token endpoint as Basic auth. + tokenRequest.Headers.Authorization = BasicHeader(c); + } + + using HttpResponseMessage tokenResponse = + await base.SendAsync(tokenRequest, cancellationToken).ConfigureAwait(false); + + if (!tokenResponse.IsSuccessStatusCode) + { + throw new RegistryAuthenticationException( + registryHost, $"the token endpoint returned {(int)tokenResponse.StatusCode} {tokenResponse.ReasonPhrase}."); + } + + TokenResponse? token; + await using (Stream s = await tokenResponse.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) + { + token = await JsonSerializer.DeserializeAsync(s, Images.OciJsonContext.TokenResponseInfo, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + if (token is null || token.ResolvedToken is null) + { + throw new InvalidAuthResponseException(registryHost, "the token endpoint response could not be parsed into a token."); + } + + return (new AuthenticationHeaderValue(BearerScheme, token.ResolvedToken), token.ResolvedExpiration); + } + + private static AuthenticationHeaderValue BasicHeader(BasicRegistryCredentials c) + => new(BasicScheme, Convert.ToBase64String(Encoding.ASCII.GetBytes($"{c.Username}:{c.Password}"))); + + private static bool TryParseChallenge(HttpResponseMessage response, out string? scheme, out AuthInfo? authInfo) + { + scheme = null; + authInfo = null; + + HttpHeaderValueCollection challenges = response.Headers.WwwAuthenticate; + if (!challenges.Any()) + { + return false; + } + + // A registry may legally return multiple WWW-Authenticate challenges (RFC 7235) in an unspecified order, + // so inspect every challenge rather than only the first. Prefer Bearer (token flow) over Basic; a Bearer + // challenge is only usable if it carries a realm. Fall back to the first Basic challenge if no usable + // Bearer is present. + AuthenticationHeaderValue? basicChallenge = null; + foreach (AuthenticationHeaderValue challenge in challenges) + { + if (challenge.Scheme is null) + { + continue; + } + + if (challenge.Scheme.Equals(BearerScheme, StringComparison.OrdinalIgnoreCase)) + { + Dictionary parsed = ParseBearerParameters(challenge.Parameter); + if (parsed.TryGetValue("realm", out string? realm)) + { + parsed.TryGetValue("service", out string? service); + parsed.TryGetValue("scope", out string? scopeValue); + scheme = challenge.Scheme; + authInfo = new AuthInfo(realm, service, scopeValue); + return true; + } + } + else if (basicChallenge is null && challenge.Scheme.Equals(BasicScheme, StringComparison.OrdinalIgnoreCase)) + { + basicChallenge = challenge; + } + } + + if (basicChallenge is not null) + { + scheme = basicChallenge.Scheme; + return true; + } + + return false; + } + + /// + /// Parses the auth-param list of a WWW-Authenticate challenge (RFC 7235 §2.1): + /// name = ( token / quoted-string ) pairs separated by commas, with optional whitespace around both + /// the = and the ,. Quoted values may contain commas and backslash-escaped characters; unquoted + /// (token) values are also accepted. Names are matched case-insensitively; if a name repeats, the first wins. + /// Malformed segments are skipped rather than aborting the whole parse. + /// + internal static Dictionary ParseBearerParameters(string? parameters) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (string.IsNullOrEmpty(parameters)) + { + return result; + } + + string s = parameters; + int i = 0; + int n = s.Length; + + while (i < n) + { + // Skip list separators and optional whitespace before the next parameter. + while (i < n && (s[i] == ',' || IsOptionalWhitespace(s[i]))) + { + i++; + } + if (i >= n) + { + break; + } + + // Parameter name: one or more token characters. + int nameStart = i; + while (i < n && IsTokenChar(s[i])) + { + i++; + } + string name = s[nameStart..i]; + + while (i < n && IsOptionalWhitespace(s[i])) + { + i++; + } + + if (name.Length == 0 || i >= n || s[i] != '=') + { + // Not a name=value pair: skip to the next separator and carry on. + SkipToNextSeparator(s, ref i); + continue; + } + + i++; // consume '=' + while (i < n && IsOptionalWhitespace(s[i])) + { + i++; + } + + string value; + if (i < n && s[i] == '"') + { + // quoted-string: read until the closing quote, honoring backslash escapes (quoted-pair). + i++; + var builder = new StringBuilder(); + while (i < n && s[i] != '"') + { + if (s[i] == '\\' && i + 1 < n) + { + i++; + } + builder.Append(s[i]); + i++; + } + i++; // consume the closing quote (harmless if it was missing) + value = builder.ToString(); + } + else + { + // token value: read until a separator or whitespace. + int valueStart = i; + while (i < n && s[i] != ',' && !IsOptionalWhitespace(s[i])) + { + i++; + } + value = s[valueStart..i]; + } + + result.TryAdd(name, value); + SkipToNextSeparator(s, ref i); + } + + return result; + } + + private static void SkipToNextSeparator(string s, ref int i) + { + while (i < s.Length && s[i] != ',') + { + i++; + } + } + + private static bool IsOptionalWhitespace(char c) => c is ' ' or '\t'; + + /// An RFC 7230 tchar: the characters allowed in a token such as a parameter name. + private static bool IsTokenChar(char c) + => char.IsAsciiLetterOrDigit(c) || c is '!' or '#' or '$' or '%' or '&' or '\'' or '*' or '+' or '-' or '.' + or '^' or '_' or '`' or '|' or '~'; + + private static async Task CloneAsync(HttpRequestMessage original) + { + var clone = new HttpRequestMessage(original.Method, original.RequestUri) + { + Version = original.Version, + VersionPolicy = original.VersionPolicy, + }; + + // Carry request options across so the authenticated retry behaves exactly like the original request. + IDictionary cloneOptions = clone.Options; + foreach (KeyValuePair option in original.Options) + { + cloneOptions[option.Key] = option.Value; + } + + foreach (KeyValuePair> header in original.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (original.Content is not null) + { + HttpContent replayContent; + + // Prefer replaying a SEEKABLE body by rewinding it — this is the real upload case (the PATCH body is a + // StreamContent over a FileStream or MemoryStream), and rewinding avoids buffering the entire blob into + // memory, which would defeat the spill-to-disk memory bound for large layers. Only a non-seekable body + // of unknown provenance falls back to buffering (small control requests, never the large layer PATCH). + Stream? contentStream = original.Content is StreamContent + ? await original.Content.ReadAsStreamAsync().ConfigureAwait(false) + : null; + + if (contentStream is { CanSeek: true }) + { + contentStream.Position = 0; + // The stream belongs to the original request (and ultimately the caller); wrap it so that + // disposing the retry clone does not close it. + replayContent = new StreamContent(new NonOwningStream(contentStream)); + } + else + { + // Fallback: buffer. Reached only for non-seekable, typically tiny, bodies. + byte[] bytes = await original.Content.ReadAsByteArrayAsync().ConfigureAwait(false); + replayContent = new ByteArrayContent(bytes); + } + + foreach (KeyValuePair> header in original.Content.Headers) + { + replayContent.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + clone.Content = replayContent; + } + + return clone; + } + + /// + /// A pass-through stream that forwards everything to an inner stream but does NOT dispose it. Used for the + /// retry clone's body so disposing the clone leaves the original request's stream — owned by the caller — + /// open and usable. + /// + internal sealed class NonOwningStream : Stream + { + private readonly Stream _inner; + + public NonOwningStream(Stream inner) => _inner = inner; + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => _inner.CanWrite; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override int Read(Span buffer) => _inner.Read(buffer); + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => _inner.ReadAsync(buffer, offset, count, cancellationToken); + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + => _inner.ReadAsync(buffer, cancellationToken); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => _inner.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => _inner.Write(buffer, offset, count); + public override void Flush() => _inner.Flush(); + + // Dispose is intentionally not overridden: the base implementation never touches _inner, which belongs to + // the caller that owns the original request. + } + + /// Parsed Bearer challenge parameters. + private sealed record AuthInfo(string Realm, string? Service, string? Scope); +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs new file mode 100644 index 000000000..bdf2f5c05 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs @@ -0,0 +1,60 @@ +namespace AWS.OCI.Builder.Registry; + +/// +/// Supplies the username/password a registry's authentication handshake needs for a given registry host. +/// +/// +/// This decouples the from where credentials come from: +/// pushes to Amazon ECR use (holding the username/password decoded +/// from a caller-supplied ECR authorization token), while pulls of public base images use +/// . +/// +public interface IRegistryCredentialSource +{ + /// + /// Returns the credentials for , or to attempt + /// anonymous access. + /// + /// The registry host, for example 123456789012.dkr.ecr.us-west-2.amazonaws.com. + /// A cancellation token. + Task GetCredentialsAsync(string registryHost, CancellationToken cancellationToken = default); +} + +/// A plain username/password pair used for registry authentication. +/// The registry username. +/// The registry password or token. +public readonly record struct BasicRegistryCredentials(string Username, string Password); + +/// +/// A credential source that always returns , causing the handshake to attempt +/// anonymous access. Suitable for pulling public base images (for example from +/// mcr.microsoft.com or Docker Hub). +/// +public sealed class AnonymousRegistryCredentialSource : IRegistryCredentialSource +{ + /// + public Task GetCredentialsAsync(string registryHost, CancellationToken cancellationToken = default) + => Task.FromResult(null); +} + +/// +/// A credential source backed by a fixed username/password pair known up front (for example one decoded from an +/// Amazon ECR authorization token). It performs no network or credential resolution and has no dependency on the +/// AWS SDK; the caller is responsible for obtaining the token and keeping it fresh for the push's duration. +/// +public sealed class StaticRegistryCredentialSource : IRegistryCredentialSource +{ + private readonly BasicRegistryCredentials _credentials; + + /// Creates a source that returns the given username/password for every host. + public StaticRegistryCredentialSource(string username, string password) + { + ArgumentNullException.ThrowIfNull(username); + ArgumentNullException.ThrowIfNull(password); + _credentials = new BasicRegistryCredentials(username, password); + } + + /// + public Task GetCredentialsAsync(string registryHost, CancellationToken cancellationToken = default) + => Task.FromResult(_credentials); +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs b/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs new file mode 100644 index 000000000..04210103e --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs @@ -0,0 +1,136 @@ +using System.Net; +using System.Net.Sockets; + +namespace AWS.OCI.Builder.Registry; + +/// +/// Validates the realm (token endpoint) URL returned in a registry's WWW-Authenticate +/// challenge before the client sends credentials to it. +/// +/// +/// A compromised or malicious registry could return a realm that points at an internal or loopback +/// address in order to exfiltrate credentials or probe the host's private network (an SSRF vector). This +/// validator rejects any realm that is not HTTPS or whose host resolves to a loopback, link-local, private, +/// or unspecified address. The approach mirrors Microsoft.NET.Build.Containers. Because we only +/// target public registries (ECR, MCR, Docker Hub), there is no insecure/on-prem exception. +/// +internal static class RealmValidator +{ + private static readonly IPNetwork[] BlockedV4Networks = + [ + IPNetwork.Parse("0.0.0.0/8"), // "this network" / unspecified + IPNetwork.Parse("10.0.0.0/8"), // private (RFC 1918) + IPNetwork.Parse("172.16.0.0/12"), // private (RFC 1918) + IPNetwork.Parse("192.168.0.0/16"), // private (RFC 1918) + IPNetwork.Parse("169.254.0.0/16"), // link-local + IPNetwork.Parse("224.0.0.0/24"), // link-local multicast + ]; + + /// + /// Validates and returns it as a . + /// + /// + /// Thrown when the realm is unparseable, not HTTPS, or points at a blocked (loopback/private/link-local) host. + /// + public static Uri Validate(string realm, string registryName) + { + if (!Uri.TryCreate(realm, UriKind.Absolute, out Uri? realmUri)) + { + throw new InvalidAuthResponseException( + registryName, $"The authentication realm '{realm}' is relative or could not be parsed."); + } + + // Only HTTPS is permitted (we target public registries exclusively). + if (!realmUri.Scheme.Equals("https", StringComparison.Ordinal)) + { + throw new InvalidAuthResponseException( + registryName, $"The authentication realm '{realm}' uses disallowed scheme '{realmUri.Scheme}'; only https is allowed."); + } + + // Use IdnHost (the ASCII/canonical host) so Unicode-dot forms that the runtime canonicalizes + // back to a loopback/private literal cannot bypass the check by appearing as a DNS name. + string realmHost = TrimTrailingDot(realmUri.IdnHost); + + if (IPAddress.TryParse(realmHost, out IPAddress? realmIp) && IsBlockedIpLiteral(realmIp)) + { + throw new InvalidAuthResponseException( + registryName, $"The authentication realm '{realm}' resolves to a blocked (loopback/private/link-local) address '{realmHost}'."); + } + + if (IsLoopbackDnsName(realmHost)) + { + // RFC 6761 reserves "localhost"/"*.localhost" for loopback resolution regardless of /etc/hosts. + throw new InvalidAuthResponseException( + registryName, $"The authentication realm '{realm}' uses a loopback host name '{realmHost}'."); + } + + // NOTE: a DNS-name realm is not resolved here. Resolving in this method would (a) be a different lookup + // from the one the socket later performs, so it would not actually defend against DNS rebinding, and + // (b) make the spelling check depend on live DNS. Instead, the ACTUAL connection IP is validated at the + // socket layer by GuardedConnectCallback (see RegistryClient.Create), which refuses to connect to any + // blocked (loopback/private/link-local) address for realm requests, the registry itself, and blob + // redirects alike — defeating rebinding because it checks the address the connection truly uses. + return realmUri; + } + + private static bool IsLoopbackDnsName(string host) + => host.Equals("localhost", StringComparison.OrdinalIgnoreCase) + || host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase); + + /// + /// Returns when 's host is a loopback name or a blocked IP + /// literal. This is a cheap pre-connect spelling check used for blob-download redirect hops; the authoritative + /// defense is applied to the real resolved connection IP at the socket layer. + /// + internal static bool IsBlockedHost(Uri uri) + { + string host = TrimTrailingDot(uri.IdnHost); + if (IsLoopbackDnsName(host)) + { + return true; + } + return IPAddress.TryParse(host, out IPAddress? literal) && IsBlockedIpLiteral(literal); + } + + /// + /// Returns when is a blocked loopback/private/link-local + /// address. Applied by the registry client's connect callback to the ACTUAL address a socket is about to + /// connect to, so it defends against DNS rebinding (the check sees the real IP, not a separately-resolved one). + /// + internal static bool IsBlockedAddress(IPAddress address) => IsBlockedIpLiteral(address); + + private static string TrimTrailingDot(string host) + => host.Length > 1 && host[^1] == '.' ? host[..^1] : host; + + private static bool IsBlockedIpLiteral(IPAddress ip) + { + if (IPAddress.IsLoopback(ip)) + { + return true; + } + + // IPv4-mapped IPv6: unwrap so we don't need a parallel set of IPv4-in-IPv6 CIDRs. + if (ip.IsIPv4MappedToIPv6) + { + return IsBlockedIpLiteral(ip.MapToIPv4()); + } + + if (ip.AddressFamily == AddressFamily.InterNetwork) + { + foreach (IPNetwork net in BlockedV4Networks) + { + if (net.Contains(ip)) + { + return true; + } + } + } + + // IPv6-only properties; all return false for IPv4 so the family gate is implicit. + return ip.Equals(IPAddress.IPv6Any) + || ip.IsIPv6LinkLocal + || ip.IsIPv6SiteLocal + || ip.IsIPv6UniqueLocal + || (ip.IsIPv6Multicast && (ip.GetAddressBytes()[1] & 0x0f) == 0x02); + } +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs new file mode 100644 index 000000000..8719e7585 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs @@ -0,0 +1,915 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Security.Cryptography; + +namespace AWS.OCI.Builder.Registry; + +/// +/// A client for the OCI Distribution / Docker Registry HTTP API (v2): pull and push of manifests and blobs. +/// +/// +/// +/// One instance targets one registry host. Authentication is handled transparently by an +/// wrapped around the supplied , so callers do not +/// deal with tokens directly. +/// +/// +/// Blob uploads send the whole blob in a single PATCH (see ). Amazon ECR does +/// not honor Content-Range offsets across multiple PATCHes to one upload session, so chunked uploads +/// are deliberately avoided. +/// +/// +public sealed class RegistryClient : IDisposable +{ + private readonly HttpClient _http; + private readonly string _registryHost; + private readonly bool _ownsHttpClient; + + /// Creates a client for using the supplied authenticating client. + /// The registry host, for example registry-1.docker.io or an ECR endpoint. + /// + /// An whose handler chain includes an . + /// Use to build one correctly. The caller owns and must dispose a client passed here. + /// + public RegistryClient(string registryHost, HttpClient httpClient) + { + _registryHost = registryHost ?? throw new ArgumentNullException(nameof(registryHost)); + _http = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _ownsHttpClient = false; + } + + private RegistryClient(string registryHost, HttpClient httpClient, bool ownsHttpClient) + { + _registryHost = registryHost; + _http = httpClient; + _ownsHttpClient = ownsHttpClient; + } + + /// The registry host this client targets. + public string RegistryHost => _registryHost; + + /// + /// Builds a whose HTTP pipeline performs the authentication handshake using + /// . + /// + /// The registry host this client targets. + /// Supplies credentials for the authentication handshake. + /// + /// The . Defaults to so a large single- + /// PATCH layer upload is bounded by the caller's cancellation token rather than cancelled by the 100-second + /// client default. Must be positive or . + /// + public static RegistryClient Create(string registryHost, IRegistryCredentialSource credentialSource, TimeSpan? timeout = null) + { + // Automatic redirects are DISABLED: the same transport also carries the token requests issued by + // AuthHandshakeHandler, and an auto-followed redirect from a validated public realm to a private/loopback + // endpoint would bypass RealmValidator (an SSRF vector). Blob downloads — which legitimately redirect to + // object storage — follow redirects explicitly in GetBlobAsync, validating each hop. + var inner = new SocketsHttpHandler + { + AutomaticDecompression = DecompressionMethods.None, + AllowAutoRedirect = false, + // SSRF defense at the socket layer: resolve the target, reject any blocked (loopback/private/link-local) + // address, and connect to the SAME address we validated — so a DNS name that resolves to a private + // address is refused, and DNS rebinding cannot swap in a different address between check and connect. + ConnectCallback = GuardedConnectAsync, + }; + var authHandler = new AuthHandshakeHandler(credentialSource, inner); + var http = new HttpClient(authHandler) + { + // No client-imposed timeout by default: a whole layer is sent in one PATCH, so the 100s default would + // cancel large uploads on slow links. The caller's cancellation token is the intended deadline. + Timeout = timeout ?? Timeout.InfiniteTimeSpan, + }; + return new RegistryClient(registryHost, http, ownsHttpClient: true); + } + + /// + /// A that resolves the target host, refuses to connect to any + /// blocked (loopback/private/link-local) address, and connects to the exact address it validated. Because the + /// check and the connect use the same resolved address, this is robust against DNS rebinding — unlike a + /// separate pre-resolution that the socket could override with a second lookup. + /// + private static async ValueTask GuardedConnectAsync(SocketsHttpConnectionContext context, CancellationToken cancellationToken) + { + string host = context.DnsEndPoint.Host; + int port = context.DnsEndPoint.Port; + + IPAddress[] addresses = IPAddress.TryParse(host, out IPAddress? literal) + ? [literal] + : await System.Net.Dns.GetHostAddressesAsync(host, cancellationToken).ConfigureAwait(false); + + // Try each resolved address in order, skipping blocked ones, connecting to the first that is allowed AND + // reachable. A blocked address is never dialed. + System.Net.Sockets.Socket? socket = null; + Exception? lastError = null; + foreach (IPAddress address in addresses) + { + if (RealmValidator.IsBlockedAddress(address)) + { + lastError = new RegistryException( + $"Refusing to connect to '{host}': it resolves to a blocked (loopback/private/link-local) address '{address}'."); + continue; + } + + var attempt = new System.Net.Sockets.Socket(address.AddressFamily, System.Net.Sockets.SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp) + { + NoDelay = true, + }; + try + { + await attempt.ConnectAsync(new IPEndPoint(address, port), cancellationToken).ConfigureAwait(false); + socket = attempt; + break; + } + catch (Exception ex) + { + lastError = ex; + attempt.Dispose(); + } + } + + if (socket is null) + { + throw lastError ?? new RegistryException($"Could not connect to '{host}:{port}'."); + } + + return new System.Net.Sockets.NetworkStream(socket, ownsSocket: true); + } + + private Uri Base => new($"https://{_registryHost}/v2/"); + + // ---- Manifests ---------------------------------------------------------------------------- + + /// + /// Fetches a manifest (or manifest list / image index) by tag or digest reference. The returned + /// carries the raw JSON, its media type, and the server-reported digest. + /// + public async Task GetManifestAsync(string repository, string reference, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(Base, $"{repository}/manifests/{reference}")); + AddManifestAcceptHeaders(request); + + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(response, $"GET manifest {repository}:{reference}", cancellationToken).ConfigureAwait(false); + + // Read the exact bytes on the wire. Digests must be computed over these, not over a string that has been + // decoded and re-encoded (which could differ by BOM/charset), so the raw bytes are exposed for verification. + byte[] raw = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + string json = System.Text.Encoding.UTF8.GetString(raw); + string mediaType = response.Content.Headers.ContentType?.MediaType + ?? throw new RegistryRequestException((int)response.StatusCode, "manifest response had no Content-Type."); + string? digest = response.Headers.TryGetValues("Docker-Content-Digest", out IEnumerable? values) + ? values.FirstOrDefault() + : null; + + return new ManifestResponse(json, raw, mediaType, digest); + } + + /// Checks whether a manifest exists for the given reference without downloading it. + public async Task ManifestExistsAsync(string repository, string reference, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Head, new Uri(Base, $"{repository}/manifests/{reference}")); + AddManifestAcceptHeaders(request); + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + return response.StatusCode == HttpStatusCode.OK; + } + + /// Uploads a manifest under the given tag or digest reference. + public async Task PutManifestAsync(string repository, string reference, string manifestJson, string mediaType, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Put, new Uri(Base, $"{repository}/manifests/{reference}")) + { + Content = new StringContent(manifestJson), + }; + request.Content.Headers.ContentType = new MediaTypeHeaderValue(mediaType); + + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(response, $"PUT manifest {repository}:{reference}", cancellationToken).ConfigureAwait(false); + } + + // ---- Blobs -------------------------------------------------------------------------------- + + /// Checks whether a blob with the given digest already exists in the repository. + public async Task BlobExistsAsync(string repository, string digest, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Head, new Uri(Base, $"{repository}/blobs/{digest}")); + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + return response.StatusCode == HttpStatusCode.OK; + } + + /// Downloads a blob by digest as a stream. Disposing the returned stream also disposes the underlying HTTP response. + /// + /// Auto-redirect is disabled on the transport (see ), so redirects to object storage are + /// followed here explicitly, up to hops. Each hop's target is validated against + /// the blocked (loopback/private/link-local) networks so a redirect cannot bounce the download onto the host's + /// private network, and the Authorization header is not carried to a different host. + /// + public async Task GetBlobAsync(string repository, string digest, CancellationToken cancellationToken = default) + { + Uri uri = new(Base, $"{repository}/blobs/{digest}"); + + for (int hop = 0; ; hop++) + { + // The request is disposed before the next loop iteration, or (on the success path below) transferred + // to the returned ResponseOwningStream so it outlives this method with the response. + var request = new HttpRequestMessage(HttpMethod.Get, uri); + if (Uri.Compare(uri, Base, UriComponents.HostAndPort, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0) + { + // Cross-host redirect hop (e.g. to object storage): never authenticate against the redirect + // target, so a malicious registry cannot redirect to a host it controls and harvest credentials + // via a 401 challenge. Presigned storage URLs carry their own authorization. + request.Options.Set(AuthHandshakeHandler.SkipAuthentication, true); + } + HttpResponseMessage response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (IsRedirect(response.StatusCode)) + { + if (hop >= MaxBlobRedirects) + { + response.Dispose(); + request.Dispose(); + throw new RegistryRequestException((int)response.StatusCode, $"blob download for {repository} {digest} exceeded {MaxBlobRedirects} redirects."); + } + + Uri? next = response.Headers.Location; + response.Dispose(); + request.Dispose(); + if (next is null) + { + throw new RegistryRequestException(302, $"blob download for {repository} {digest} returned a redirect with no Location."); + } + if (!next.IsAbsoluteUri) + { + next = new Uri(uri, next); + } + if (!next.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryRequestException(302, $"blob download for {repository} {digest} redirected to a non-https URL."); + } + // SSRF guard: never follow a redirect onto a blocked (loopback/private/link-local) address. + if (RealmValidator.IsBlockedHost(next)) + { + throw new RegistryRequestException(302, $"blob download for {repository} {digest} redirected to a blocked (loopback/private/link-local) host '{next.Host}'."); + } + uri = next; + continue; + } + + try + { + await EnsureSuccessAsync(response, $"GET blob {repository} {digest}", cancellationToken).ConfigureAwait(false); + Stream body = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + // Hand ownership of the response (and its request) to the caller via the returned stream: disposing + // the stream disposes the response, so the HttpResponseMessage/HttpRequestMessage are never leaked. + return new ResponseOwningStream(body, response, request); + } + catch + { + response.Dispose(); + request.Dispose(); + throw; + } + } + } + + private const int MaxBlobRedirects = 5; + + private static bool IsRedirect(HttpStatusCode status) + => status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther + or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect; + + /// + /// A read-through stream wrapper that owns the (and the + /// ) the content came from. Disposing this stream disposes the inner content + /// stream and then the response and request, so callers that using/await using the stream + /// returned by never leak the HTTP message pair. + /// + private sealed class ResponseOwningStream : Stream + { + private readonly Stream _inner; + private readonly HttpResponseMessage _response; + private readonly HttpRequestMessage _request; + + public ResponseOwningStream(Stream inner, HttpResponseMessage response, HttpRequestMessage request) + { + _inner = inner; + _response = response; + _request = request; + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override int Read(Span buffer) => _inner.Read(buffer); + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => _inner.ReadAsync(buffer, offset, count, cancellationToken); + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + => _inner.ReadAsync(buffer, cancellationToken); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override void Flush() => _inner.Flush(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _inner.Dispose(); + _response.Dispose(); + _request.Dispose(); + } + base.Dispose(disposing); + } + + public override async ValueTask DisposeAsync() + { + await _inner.DisposeAsync().ConfigureAwait(false); + _response.Dispose(); + _request.Dispose(); + await base.DisposeAsync().ConfigureAwait(false); + } + } + + /// + /// Attempts a cross-repository blob mount so an already-present base-image layer is not re-uploaded. + /// Returns when the registry mounted the blob (HTTP 201), + /// when it requires a full upload instead (HTTP 202). + /// + public async Task TryMountBlobAsync(string repository, string digest, string fromRepository, CancellationToken cancellationToken = default) + { + var uri = new Uri(Base, $"{repository}/blobs/uploads/?mount={Uri.EscapeDataString(digest)}&from={Uri.EscapeDataString(fromRepository)}"); + using var request = new HttpRequestMessage(HttpMethod.Post, uri); + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.Created) + { + return true; // Mounted — no upload needed. + } + if (response.StatusCode == HttpStatusCode.Accepted) + { + return false; // Not mounted; caller must upload. + } + + await EnsureSuccessAsync(response, $"POST blob mount {repository} {digest}", cancellationToken).ConfigureAwait(false); + return false; + } + + /// + /// Uploads a blob using the POST/PATCH/PUT flow, sending the whole blob in a single + /// PATCH. Skips the upload entirely if the blob already exists. + /// + /// + /// + /// The blob is sent as one PATCH rather than multiple 5 MiB chunks. Amazon ECR does not honor the + /// Content-Range offset across successive PATCHes to the same upload session — each is treated as + /// writing from offset 0, so only the final chunk would survive and the commit would fail the digest check. + /// A single PATCH of the full content is what ECR expects and is what registries accept generally. + /// + /// + /// The request advertises a Content-Length (from , or the stream's + /// when it is seekable). This is important for memory: when the length is known + /// streams straight onto the socket, whereas without a + /// known length it must buffer the entire body into memory first to compute the length — which would + /// defeat the spill-to-disk download path by re-materializing a large layer in memory on the upload side. + /// + /// + /// The destination repository (for example myapp). + /// The sha256:... digest of the (compressed) blob content. + /// The blob bytes. + /// A cancellation token. + /// + /// Optional callback invoked as bytes are read from and sent, receiving the + /// cumulative number of bytes sent so far. Called on the thread driving the upload; keep it cheap and + /// non-throwing. When the blob already exists no bytes are sent and this is not called. + /// + /// + /// The exact length of in bytes, when the caller knows it. Supplying it lets the + /// upload stream without buffering the whole body in memory. When , a seekable stream's + /// is used; a non-seekable stream of unknown length falls back to buffering. + /// + public async Task UploadBlobAsync( + string repository, + string digest, + Stream content, + CancellationToken cancellationToken = default, + Action? onBytesSent = null, + long? contentLength = null) + { + if (await BlobExistsAsync(repository, digest, cancellationToken).ConfigureAwait(false)) + { + return; + } + + // 1. Start an upload session; the registry returns the upload URL in the Location header. + Uri uploadUri = await StartUploadAsync(repository, cancellationToken).ConfigureAwait(false); + + // 2. PATCH the entire blob in one request. (See remarks: ECR mishandles multi-PATCH chunking.) When a + // progress callback is supplied, wrap the source so bytes are reported as the HTTP stack reads them. + Stream patchContent = onBytesSent is null ? content : new ProgressReportingStream(content, onBytesSent); + + // Resolve the body length: prefer the caller-supplied value, else a seekable stream's REMAINING length + // (Length - Position). StreamContent sends from the current position, so advertising the full Length for a + // stream positioned past the start would claim more bytes than are actually sent and fail the upload. + long? bodyLength = contentLength ?? (content.CanSeek ? content.Length - content.Position : null); + + using (var patch = new HttpRequestMessage(HttpMethod.Patch, uploadUri) + { + Content = new StreamContent(patchContent), + }) + { + patch.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + if (bodyLength is { } length) + { + // Advertising the length switches HttpClient from "buffer-to-measure" to streaming. + patch.Content.Headers.ContentLength = length; + } + + using HttpResponseMessage patchResponse = await _http.SendAsync(patch, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(patchResponse, $"PATCH blob {repository}", cancellationToken).ConfigureAwait(false); + + // The registry hands back the upload URL to use for the commit. + uploadUri = LocationFrom(patchResponse) ?? uploadUri; + } + + // 3. Close the session with PUT ?digest=..., which commits the blob. + Uri commitUri = AppendQuery(uploadUri, "digest", digest); + using var put = new HttpRequestMessage(HttpMethod.Put, commitUri) + { + Content = new ByteArrayContent(Array.Empty()), + }; + put.Content.Headers.ContentLength = 0; + using HttpResponseMessage putResponse = await _http.SendAsync(put, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(putResponse, $"PUT blob commit {repository} {digest}", cancellationToken).ConfigureAwait(false); + } + + private async Task StartUploadAsync(string repository, CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage(HttpMethod.Post, new Uri(Base, $"{repository}/blobs/uploads/")); + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(response, $"POST blob upload start {repository}", cancellationToken).ConfigureAwait(false); + + return LocationFrom(response) + ?? throw new RegistryRequestException((int)response.StatusCode, "the registry did not return an upload Location header."); + } + + // ---- Helpers ------------------------------------------------------------------------------ + + private static void AddManifestAcceptHeaders(HttpRequestMessage request) + { + foreach (string mediaType in new[] + { + MediaTypes.OciManifestV1, + MediaTypes.OciImageIndexV1, + MediaTypes.DockerManifestV2, + MediaTypes.DockerManifestListV2, + }) + { + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(mediaType)); + } + } + + private Uri? LocationFrom(HttpResponseMessage response) + { + if (response.Headers.Location is not { } location) + { + return null; + } + // A registry may return a relative Location; resolve it against the registry base. + return location.IsAbsoluteUri ? location : new Uri(new Uri($"https://{_registryHost}"), location); + } + + private static Uri AppendQuery(Uri uri, string key, string value) + { + var builder = new UriBuilder(uri); + string encoded = $"{Uri.EscapeDataString(key)}={Uri.EscapeDataString(value)}"; + builder.Query = string.IsNullOrEmpty(builder.Query) ? encoded : builder.Query.TrimStart('?') + "&" + encoded; + return builder.Uri; + } + + private static async Task EnsureSuccessAsync(HttpResponseMessage response, string operation, CancellationToken cancellationToken) + { + if (response.IsSuccessStatusCode) + { + return; + } + + string body = string.Empty; + try + { + body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + // Ignore body read failures; the status code is the primary signal. + } + + throw new RegistryRequestException( + (int)response.StatusCode, + $"{operation} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {Truncate(body, 512)}"); + } + + private static string Truncate(string value, int max) + => value.Length <= max ? value : value[..max] + "…"; + + /// Computes the sha256:... digest of a stream (used by callers constructing blobs). + /// + /// The digest covers the bytes from the stream's current position to its end, and the stream is left + /// positioned at the end — the same semantics as . To digest a seekable + /// stream's full content, set to 0 before calling. The position is not + /// reset automatically, so callers can digest a suffix intentionally and non-seekable streams are supported. + /// + /// The stream to hash, read from its current position to the end. + public static string ComputeDigest(Stream content) + { + byte[] hash = SHA256.HashData(content); + // Convert.ToHexStringLower exists only on net9.0+, so lower-case explicitly for net8.0 compatibility. + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + /// Computes the sha256:... digest of a byte buffer. + public static string ComputeDigest(ReadOnlySpan content) + { + Span hash = stackalloc byte[SHA256.HashSizeInBytes]; + SHA256.HashData(content, hash); + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + /// + /// Verifies that hashes to , throwing + /// otherwise. The comparison is case-insensitive on the hex and + /// tolerant of an absent sha256: prefix on the expected value. Only the sha256 algorithm is + /// supported; an expected digest naming a different algorithm is rejected. + /// + /// A short description of the content, used in the error message (e.g. "config blob"). + /// The digest the content is expected to have (sha256:<hex>). + /// The downloaded bytes to verify. + public static void VerifyDigest(string what, string expectedDigest, ReadOnlySpan content) + { + ArgumentException.ThrowIfNullOrWhiteSpace(expectedDigest); + + int colon = expectedDigest.IndexOf(':'); + if (colon >= 0) + { + string algorithm = expectedDigest[..colon]; + if (!algorithm.Equals("sha256", StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryException( + $"Cannot verify {what}: unsupported digest algorithm '{algorithm}' (only sha256 is supported)."); + } + } + + string actual = ComputeDigest(content); + // Normalize both to "sha256:" for comparison. + string expectedNormalized = colon >= 0 + ? "sha256:" + expectedDigest[(colon + 1)..].ToLowerInvariant() + : "sha256:" + expectedDigest.ToLowerInvariant(); + + if (!actual.Equals(expectedNormalized, StringComparison.Ordinal)) + { + throw new DigestMismatchException(what, expectedNormalized, actual); + } + } + + /// + /// Downloads a blob by digest, buffers it fully, and verifies that the bytes hash to the requested digest + /// before returning them. Use this (rather than ) whenever the expected digest is + /// known, so corrupted or tampered content is rejected rather than trusted. + /// + /// The repository to read from. + /// The expected sha256:... digest; the download is verified against it. + /// A short description used in any integrity error (e.g. "base layer"). + /// + /// The size declared by the manifest descriptor, if known. When supplied, the downloaded length must match it. + /// + /// A cancellation token. + /// The downloaded bytes do not hash to . + /// The downloaded length differs from . + public async Task GetVerifiedBlobBytesAsync(string repository, string digest, string what, long? expectedSize = null, CancellationToken cancellationToken = default) + { + byte[] bytes; + await using (Stream stream = await GetBlobAsync(repository, digest, cancellationToken).ConfigureAwait(false)) + { + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + bytes = buffer.ToArray(); + } + + VerifyDigest(what, digest, bytes); + VerifySize(what, digest, expectedSize, bytes.LongLength); + return bytes; + } + + /// + /// Downloads a blob by digest to a temporary file on disk (rather than buffering it in memory), verifying its + /// integrity twice: once against the bytes as they stream off the network, and again by re-reading the file + /// from disk before it is handed back. Use this for large layers where an in-memory buffer would be a memory + /// problem; small blobs should use . + /// + /// + /// + /// The second (re-read) verification is deliberate: the on-the-wire hash proves the download was not + /// corrupted, but the bytes could still be corrupted after that by a faulty disk write (a bit flip, + /// a torn or truncated write). Re-hashing the file as read back from disk — which is what will actually be + /// uploaded — closes that window, so a corrupt spilled layer is rejected here rather than committed to the + /// registry. + /// + /// + /// The returned handle owns the temporary file and an open read-only stream positioned at the start, ready to + /// pass to . Disposing the handle closes the stream and deletes the temp file; + /// the file is also cleaned up if verification fails. + /// + /// + /// The repository to read from. + /// The expected sha256:... digest; both verifications check against it. + /// A short description used in any integrity error (e.g. "base layer"). + /// + /// The size declared by the manifest descriptor, if known. When supplied, the downloaded length must match it. + /// + /// A cancellation token. + /// + /// The downloaded bytes, or the bytes as read back from disk, do not hash to . + /// + /// The downloaded length differs from . + public async Task GetVerifiedBlobToFileAsync(string repository, string digest, string what, long? expectedSize = null, CancellationToken cancellationToken = default) + { + // A unique temp path under the system temp dir (honors TMPDIR). Created now so we can always clean it up. + string tempPath = Path.Combine(Path.GetTempPath(), $"aws-oci-layer-{Guid.NewGuid():N}.tmp"); + + try + { + // 1. Stream the blob to disk, hashing (and counting) the exact bytes written as we go. + string downloadedDigest; + long downloadedLength = 0; + await using (Stream source = await GetBlobAsync(repository, digest, cancellationToken).ConfigureAwait(false)) + await using (var file = SpillFile.CreateWrite(tempPath)) + using (var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256)) + { + byte[] rented = System.Buffers.ArrayPool.Shared.Rent(81920); + try + { + int read; + while ((read = await source.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0) + { + hasher.AppendData(rented, 0, read); + downloadedLength += read; + await file.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false); + } + } + finally + { + System.Buffers.ArrayPool.Shared.Return(rented); + } + + // Push the OS buffers to disk before we re-read, so the re-read actually exercises the stored bytes. + await file.FlushAsync(cancellationToken).ConfigureAwait(false); + file.Flush(flushToDisk: true); + + downloadedDigest = "sha256:" + Convert.ToHexString(hasher.GetHashAndReset()).ToLowerInvariant(); + } + + VerifyDigest(what, digest, downloadedDigest); + VerifySize(what, digest, expectedSize, downloadedLength); + + // 2. Re-read the file from disk and verify again — catches a faulty disk write that corrupted the + // spilled bytes after the download-side hash. + await VerifyFileDigestAsync(what + " (spilled to disk)", digest, tempPath, cancellationToken).ConfigureAwait(false); + + // 3. Hand back an open read stream over the verified file. The handle owns deletion. + var readStream = new FileStream( + tempPath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, useAsync: true); + return new VerifiedBlobFile(tempPath, readStream); + } + catch + { + TryDeleteFile(tempPath); + throw; + } + } + + /// + /// Verifies that a downloaded blob's length matches the size its manifest descriptor declared, throwing + /// on mismatch. A wrong declared size would otherwise surface much + /// later as an opaque Content-Length transport error during the push, or as a manifest referencing an + /// incorrect size. No check is made when is . + /// + internal static void VerifySize(string what, string digest, long? expectedSize, long actualSize) + { + if (expectedSize is { } declared && declared != actualSize) + { + throw new BlobSizeMismatchException(what, digest, declared, actualSize); + } + } + + /// + /// Verifies that an expected digest matches an already-computed digest string (both normalized), throwing + /// on mismatch. Used when the content was hashed incrementally while + /// streaming, so there is no in-memory buffer to re-hash. + /// + private static void VerifyDigest(string what, string expectedDigest, string actualDigest) + { + ArgumentException.ThrowIfNullOrWhiteSpace(expectedDigest); + + int colon = expectedDigest.IndexOf(':'); + if (colon >= 0 && !expectedDigest[..colon].Equals("sha256", StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryException( + $"Cannot verify {what}: unsupported digest algorithm '{expectedDigest[..colon]}' (only sha256 is supported)."); + } + + string expectedNormalized = colon >= 0 + ? "sha256:" + expectedDigest[(colon + 1)..].ToLowerInvariant() + : "sha256:" + expectedDigest.ToLowerInvariant(); + + if (!actualDigest.Equals(expectedNormalized, StringComparison.Ordinal)) + { + throw new DigestMismatchException(what, expectedNormalized, actualDigest); + } + } + + /// Hashes a file's contents from disk and verifies the digest, without buffering the whole file. + private static async Task VerifyFileDigestAsync(string what, string expectedDigest, string path, CancellationToken cancellationToken) + { + string actual; + await using (var file = new FileStream( + path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, useAsync: true)) + { + byte[] hash = await SHA256.HashDataAsync(file, cancellationToken).ConfigureAwait(false); + actual = "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + VerifyDigest(what, expectedDigest, actual); + } + + private static void TryDeleteFile(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch + { + // Best-effort cleanup; a leftover temp file is not worth masking the original error. + } + } + + /// + /// Disposes the underlying only when this instance created it (via + /// ). A client supplied through the public constructor is left for its owner to dispose. + /// + public void Dispose() + { + if (_ownsHttpClient) + { + _http.Dispose(); + } + } +} + +/// The result of fetching a manifest: its raw JSON, media type, and server-reported digest. +/// The manifest JSON decoded as UTF-8 (convenient for parsing). +/// The exact bytes received on the wire — digests must be computed over these. +/// The manifest media type (OCI or Docker; single image or index). +/// The server-reported Docker-Content-Digest, if present. +public readonly record struct ManifestResponse(string Json, byte[] RawBytes, string MediaType, string? Digest); + +/// +/// A verified blob that has been spilled to a temporary file on disk. Owns both the open read stream (positioned +/// at the start, ready to upload) and the temp file; disposing it closes the stream and deletes the file. +/// +public sealed class VerifiedBlobFile : IAsyncDisposable, IDisposable +{ + private readonly string _path; + private bool _disposed; + + internal VerifiedBlobFile(string path, FileStream stream) + { + _path = path; + Stream = stream; + } + + /// The open, read-only stream over the verified file, positioned at offset 0. + public FileStream Stream { get; } + + /// The full path of the temporary file backing . + public string Path => _path; + + /// Closes the stream and deletes the temporary file. + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + _disposed = true; + await Stream.DisposeAsync().ConfigureAwait(false); + Delete(); + } + + /// Closes the stream and deletes the temporary file. + public void Dispose() + { + if (_disposed) + { + return; + } + _disposed = true; + Stream.Dispose(); + Delete(); + } + + private void Delete() + { + try + { + if (File.Exists(_path)) + { + File.Delete(_path); + } + } + catch + { + // Best-effort cleanup. + } + } +} + +/// +/// A read-only pass-through stream that reports cumulative bytes read to a callback. Wrapping the upload source +/// in this lets the registry client surface byte-level upload progress: the HTTP stack reads from it to send the +/// PATCH body, and each read advances the reported total. The wrapped stream is not disposed by this wrapper — +/// its owner retains responsibility for it. +/// +internal sealed class ProgressReportingStream : Stream +{ + private readonly Stream _inner; + private readonly Action _onBytesRead; + private long _totalRead; + + public ProgressReportingStream(Stream inner, Action onBytesRead) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _onBytesRead = onBytesRead ?? throw new ArgumentNullException(nameof(onBytesRead)); + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + + public override long Position + { + get => _inner.Position; + set => _inner.Position = value; + } + + public override int Read(byte[] buffer, int offset, int count) + { + int read = _inner.Read(buffer, offset, count); + Report(read); + return read; + } + + public override int Read(Span buffer) + { + int read = _inner.Read(buffer); + Report(read); + return read; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int read = await _inner.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + Report(read); + return read; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int read = await _inner.ReadAsync(buffer.AsMemory(offset, count), cancellationToken).ConfigureAwait(false); + Report(read); + return read; + } + + private void Report(int read) + { + if (read <= 0) + { + return; + } + _totalRead += read; + _onBytesRead(_totalRead); + } + + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override void Flush() => _inner.Flush(); +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs new file mode 100644 index 000000000..5586c5630 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs @@ -0,0 +1,99 @@ +namespace AWS.OCI.Builder.Registry; + +/// +/// The base type for errors raised while interacting with a container registry — authentication, +/// malformed responses, failed requests, or higher-level build problems such as a missing platform. +/// +public class RegistryException : Exception +{ + /// Creates the exception with a message. + public RegistryException(string message) : base(message) { } + + /// Creates the exception with a message and an inner cause. + public RegistryException(string message, Exception innerException) : base(message, innerException) { } +} + +/// Thrown when a registry's authentication challenge or token response is malformed or unsafe. +public sealed class InvalidAuthResponseException : RegistryException +{ + /// The registry that produced the invalid response. + public string RegistryName { get; } + + /// Creates the exception. + public InvalidAuthResponseException(string registryName, string message) + : base($"Invalid authentication response from registry '{registryName}': {message}") + => RegistryName = registryName; +} + +/// Thrown when the client cannot authenticate to, or is denied access by, a registry. +public sealed class RegistryAuthenticationException : RegistryException +{ + /// The registry that denied access. + public string RegistryName { get; } + + /// Creates the exception. + public RegistryAuthenticationException(string registryName, string message) + : base($"Unable to authenticate to registry '{registryName}': {message}") + => RegistryName = registryName; +} + +/// Thrown when a registry request fails with an unexpected HTTP status. +public sealed class RegistryRequestException : RegistryException +{ + /// The HTTP status code returned by the registry. + public int StatusCode { get; } + + /// Creates the exception. + public RegistryRequestException(int statusCode, string message) + : base(message) + => StatusCode = statusCode; +} + +/// +/// Thrown when downloaded content does not hash to its expected digest — i.e. a content-integrity check failed. +/// This indicates the registry, a cache/CDN, or the transport returned bytes that do not match the +/// content-addressable digest they were requested under, and the content must not be trusted. +/// +public sealed class DigestMismatchException : RegistryException +{ + /// The digest the content was expected to have. + public string ExpectedDigest { get; } + + /// The digest actually computed over the downloaded bytes. + public string ActualDigest { get; } + + /// Creates the exception. + public DigestMismatchException(string what, string expectedDigest, string actualDigest) + : base($"Integrity check failed for {what}: expected {expectedDigest} but the downloaded content hashed to {actualDigest}.") + { + ExpectedDigest = expectedDigest; + ActualDigest = actualDigest; + } +} + +/// +/// Thrown when a downloaded blob's length does not match the size declared for it in a manifest descriptor. +/// The content may still hash to the expected digest — the descriptor itself is wrong — but pushing it would +/// fail (the upload's Content-Length could not match) or produce a manifest referencing an incorrect size, so +/// the mismatch is reported up front with a clear error instead. +/// +public sealed class BlobSizeMismatchException : RegistryException +{ + /// The digest of the blob whose size did not match. + public string Digest { get; } + + /// The size, in bytes, declared by the manifest descriptor. + public long DeclaredSize { get; } + + /// The actual size, in bytes, of the downloaded blob. + public long ActualSize { get; } + + /// Creates the exception. + public BlobSizeMismatchException(string what, string digest, long declaredSize, long actualSize) + : base($"Integrity check failed for {what} {digest}: the manifest declares a size of {declaredSize} bytes but the blob is {actualSize} bytes.") + { + Digest = digest; + DeclaredSize = declaredSize; + ActualSize = actualSize; + } +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs b/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs new file mode 100644 index 000000000..ad77dfb87 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs @@ -0,0 +1,40 @@ +namespace AWS.OCI.Builder.Registry; + +/// +/// Helpers for creating the temporary "spill" files this library writes layer/blob bytes to, with owner-only +/// access on Unix. +/// +/// +/// These files land in the shared system temp directory. With the usual umask (022) a newly created file is +/// world-readable (0644), so another local user could read a private image layer — including files a manifest +/// mapped as 0600 — for the lifetime of the handle. The permissions recorded inside the tar do not +/// protect the outer spill file. Creating the file 0600 from the start (atomically, via +/// ) closes that window. On Windows the mode is not applicable and +/// is left unset; default ACLs already restrict the per-user temp directory. +/// +internal static class SpillFile +{ + // Owner read+write only (0600). + private const UnixFileMode OwnerOnly = UnixFileMode.UserRead | UnixFileMode.UserWrite; + + /// + /// Opens a new file for writing with owner-only permissions on Unix. Fails if the file already exists + /// (), matching the unique-temp-name contract the callers rely on. + /// + public static FileStream CreateWrite(string path, int bufferSize = 81920) + { + var options = new FileStreamOptions + { + Mode = FileMode.CreateNew, + Access = FileAccess.Write, + Share = FileShare.None, + BufferSize = bufferSize, + Options = FileOptions.Asynchronous, + }; + if (!OperatingSystem.IsWindows()) + { + options.UnixCreateMode = OwnerOnly; + } + return new FileStream(path, options); + } +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs b/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs new file mode 100644 index 000000000..6bef6e3b9 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs @@ -0,0 +1,43 @@ +using System.Text.Json.Serialization; + +namespace AWS.OCI.Builder.Registry; + +/// The JSON body returned by a registry token endpoint during the Bearer auth handshake. +/// +/// Internal and top-level (rather than nested in ) so the source-generated +/// can reference it — a source-gen context cannot target a private nested type. +/// +internal sealed record TokenResponse +{ + [JsonPropertyName("token")] + public string? Token { get; init; } + + [JsonPropertyName("access_token")] + public string? AccessToken { get; init; } + + [JsonPropertyName("expires_in")] + public int? ExpiresIn { get; init; } + + [JsonPropertyName("issued_at")] + public DateTimeOffset? IssuedAt { get; init; } + + /// The usable token (registries populate either token or access_token). + public string? ResolvedToken => Token ?? AccessToken; + + /// + /// The computed expiry. Per the token spec, default to 60 seconds of validity when the endpoint + /// omits expires_in; subtract a small safety margin so a cached token is refreshed before it + /// actually expires. + /// + public DateTimeOffset ResolvedExpiration + { + get + { + DateTimeOffset issued = IssuedAt ?? DateTimeOffset.UtcNow; + int validity = ExpiresIn ?? 60; + // 10-second safety margin, floored so very short tokens still move forward in time. + int margin = Math.Min(10, Math.Max(0, validity - 1)); + return issued.AddSeconds(validity - margin); + } + } +} diff --git a/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs b/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs new file mode 100644 index 000000000..0c7d98697 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs @@ -0,0 +1,79 @@ +namespace AWS.OCI.Builder; + +/// +/// Helpers for expressing Unix file permissions the way chmod does — in octal — and converting them to the +/// flags the layer builder uses. +/// +/// +/// C# has no octal integer literal, so the familiar 0755 cannot be written directly (and 0755 in C# +/// would be the decimal 755, the wrong bits). Use with "755", or +/// where each decimal digit is read as one octal permission digit (so the C# literal +/// 755 means octal 0755). Each of the three permission digits is a 0–7 value whose bits are +/// read(4)/write(2)/execute(1) for owner, group, and other respectively. +/// +public static class UnixPermissions +{ + /// + /// Parses an octal permission string such as "755", "0644", or "0o700" into a + /// . A leading 0 or 0o prefix is accepted and ignored. Only the nine + /// standard permission bits are supported; set-uid/set-gid/sticky bits (a fourth leading digit other than 0) + /// are rejected. + /// + /// Three octal digits, optionally prefixed with 0 or 0o. + /// The value is null/empty or not valid octal permission notation. + public static UnixFileMode FromOctal(string octal) + { + ArgumentException.ThrowIfNullOrWhiteSpace(octal); + + string digits = octal.Trim(); + // Accept a leading "0o"/"0O" prefix, or a single leading "0". + if (digits.StartsWith("0o", StringComparison.OrdinalIgnoreCase)) + { + digits = digits[2..]; + } + else if (digits.Length == 4 && digits[0] == '0') + { + digits = digits[1..]; + } + + if (digits.Length != 3) + { + throw new ArgumentException( + $"'{octal}' is not a valid octal permission. Use three octal digits, e.g. \"755\" or \"0644\" " + + "(set-uid/set-gid/sticky bits are not supported).", + nameof(octal)); + } + + int value = 0; + foreach (char c in digits) + { + if (c is < '0' or > '7') + { + throw new ArgumentException( + $"'{octal}' contains '{c}', which is not an octal digit (0–7).", nameof(octal)); + } + value = (value << 3) | (c - '0'); + } + + // The nine low bits map directly onto UnixFileMode, which uses the same rwx-per-triad layout. + return (UnixFileMode)value; + } + + /// + /// Parses a permission value whose decimal digits are read as octal — so the C# literal 755 + /// means octal 0755, and 644 means 0644. This lets callers write the number they would + /// type into chmod without a string. Equivalent to FromOctal(value.ToString()). + /// + /// A value in the range 0–777 whose decimal digits are each 0–7. + /// The value is out of range or has a digit outside 0–7. + public static UnixFileMode FromOctal(int octalDigits) + { + if (octalDigits is < 0 or > 777) + { + throw new ArgumentException( + $"{octalDigits} is out of range; pass the chmod-style digits, e.g. 755 or 644 (0–777).", + nameof(octalDigits)); + } + return FromOctal(octalDigits.ToString("D3", System.Globalization.CultureInfo.InvariantCulture)); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.IntegrationTests/AWS.OCI.Builder.IntegrationTests.csproj b/Libraries/test/AWS.OCI.Builder.IntegrationTests/AWS.OCI.Builder.IntegrationTests.csproj new file mode 100644 index 000000000..2578b3bb6 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.IntegrationTests/AWS.OCI.Builder.IntegrationTests.csproj @@ -0,0 +1,27 @@ + + + + + + $(DefaultPackageTargets) + false + + false + false + enable + enable + + + + + + + + + + + + + + + diff --git a/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs b/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs new file mode 100644 index 000000000..094b131ee --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs @@ -0,0 +1,376 @@ +using Amazon; +using Amazon.ECR; +using Amazon.ECR.Model; +using AWS.OCI.Builder; +using Xunit; +using Xunit.Abstractions; + +namespace AWS.OCI.Builder.IntegrationTests; + +/// +/// End-to-end integration test that pushes a real image to Amazon ECR using the library's public API, then +/// verifies the image landed by querying ECR. AWS credentials and region are taken from the environment (the AWS SDK default chain); set +/// AWS_SHARED_CREDENTIALS_FILE / AWS_PROFILE / AWS_REGION as needed before running. +/// +/// +/// What it does, end to end: +/// +/// Ensures a dedicated ECR repository exists (creates it if missing). +/// Pulls a small public base image, overlays a generated file as a new layer, and pushes to the repo +/// under a unique tag via . +/// Calls ECR DescribeImages to confirm the pushed tag exists and its digest matches what +/// returned. +/// Cleans up by deleting just the tag it pushed (the repository is left in place). +/// +/// +/// Overridable via environment variables: OCI_ECR_REPOSITORY (default aws-oci-builder-itest), +/// OCI_ECR_REGION (default us-west-2), OCI_ECR_BASE_IMAGE +/// (default mcr.microsoft.com/dotnet/runtime-deps:8.0). +/// +/// +[Trait("Category", "Integration")] +public class EcrPushIntegrationTests +{ + private const string DefaultRepository = "aws-oci-builder-itest"; + private const string DefaultRegion = "us-west-2"; + private const string DefaultBaseImage = "mcr.microsoft.com/dotnet/runtime-deps:8.0"; + private const string Platform = "linux/amd64"; + + private readonly ITestOutputHelper _output; + + public EcrPushIntegrationTests(ITestOutputHelper output) => _output = output; + + [Fact] + public async Task Pushes_image_to_real_ecr_and_verifies_it() + { + string repository = EnvOr("OCI_ECR_REPOSITORY", DefaultRepository); + string regionName = EnvOr("OCI_ECR_REGION", DefaultRegion); + string baseImage = EnvOr("OCI_ECR_BASE_IMAGE", DefaultBaseImage); + RegionEndpoint region = RegionEndpoint.GetBySystemName(regionName); + string tag = "itest-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss") + "-" + Guid.NewGuid().ToString("N")[..6]; + + using var ecr = new AmazonECRClient(region); + + // ---- 1. Resolve the registry host and ensure the repository exists. ---- + string accountId = await GetAccountIdAsync(ecr); + string registryHost = $"{accountId}.dkr.ecr.{regionName}.amazonaws.com"; + string targetImage = $"{registryHost}/{repository}:{tag}"; + _output.WriteLine($"Target: {targetImage}"); + + await EnsureRepositoryAsync(ecr, repository); + + string? pushedDigest = null; + try + { + // ---- 2. Build from the base image + a generated layer, and push. ---- + string payloadFile = Path.Combine(Path.GetTempPath(), $"oci-itest-{Guid.NewGuid():N}.txt"); + await File.WriteAllTextAsync(payloadFile, "Hello from the AWS.OCI.Builder integration test.\n"); + + try + { + using OciImageBuilder builder = await OciImageBuilder.FromBaseImageAsync(baseImage, Platform); + await builder.AddLayerAsync( + new FileManifest { { payloadFile, "/app/hello.txt" } }, + createdBy: "AWS.OCI.Builder integration test"); + builder.Config.SetWorkingDirectory("/app"); + builder.Config.SetLabel("com.amazonaws.oci-builder.itest", tag); + + // Region is inferred from the ECR host in the target; the auth token is fetched via the SDK. + string authToken = await GetAuthorizationTokenStringAsync(ecr); + + // Collect push progress so we can assert the event contract against real ECR byte streaming. Use a + // SYNCHRONOUS sink: Progress marshals callbacks asynchronously, so completing PushToEcrAsync would + // not guarantee every Completed event has arrived, making the assertions below racy. A synchronous + // sink reports inline on the push thread, so all events are present once the push returns. + var events = new List(); + var options = new EcrPushOptions + { + Progress = new SynchronousProgress(events.Add), + }; + pushedDigest = await builder.PushToEcrAsync(targetImage, authToken, options); + + // Verify the progress contract end to end. All events are already delivered (synchronous sink). + List seen = events; + + Assert.NotEmpty(seen); + int totalLayers = seen[0].TotalLayers; + Assert.All(seen, e => Assert.Equal(totalLayers, e.TotalLayers)); + Assert.All(seen, e => Assert.InRange(e.LayerIndex, 1, totalLayers)); + + // Every layer (1..totalLayers) terminates on exactly one Completed event. + for (int index = 1; index <= totalLayers; index++) + { + LayerPushProgress completed = Assert.Single( + seen, e => e.LayerIndex == index && e.Phase == LayerPushPhase.Completed); + // A non-skipped layer's byte count reaches its size; a skipped one reports 0 of a known size. + if (completed.IsSkipped) + { + Assert.Equal(0, completed.BytesPushed); + } + else + { + Assert.Equal(completed.LayerSizeBytes, completed.BytesPushed); + } + } + _output.WriteLine($"Progress: {seen.Count} events across {totalLayers} layers."); + } + finally + { + File.Delete(payloadFile); + } + + _output.WriteLine($"Pushed digest: {pushedDigest}"); + Assert.StartsWith("sha256:", pushedDigest); + + // ---- 3. Verify via ECR that the tag exists and the digest matches. ---- + DescribeImagesResponse described = await ecr.DescribeImagesAsync(new DescribeImagesRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + + ImageDetail detail = Assert.Single(described.ImageDetails); + Assert.Contains(tag, detail.ImageTags); + Assert.Equal(pushedDigest, detail.ImageDigest); + _output.WriteLine($"Verified image {detail.ImageDigest} ({detail.ImageSizeInBytes} bytes) in ECR."); + } + finally + { + // ---- 4. Best-effort cleanup: delete only the tag this test pushed. ---- + try + { + await ecr.BatchDeleteImageAsync(new BatchDeleteImageRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + _output.WriteLine($"Cleaned up tag {tag}."); + } + catch (Exception ex) + { + _output.WriteLine($"Cleanup of tag {tag} failed (non-fatal): {ex.Message}"); + } + } + } + + [Fact] + public async Task Pushes_image_to_real_ecr_using_the_spill_to_disk_path_for_large_base_layers() + { + string repository = EnvOr("OCI_ECR_REPOSITORY", DefaultRepository); + string regionName = EnvOr("OCI_ECR_REGION", DefaultRegion); + string baseImage = EnvOr("OCI_ECR_BASE_IMAGE", DefaultBaseImage); + RegionEndpoint region = RegionEndpoint.GetBySystemName(regionName); + string tag = "itest-spill-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss") + "-" + Guid.NewGuid().ToString("N")[..6]; + + using var ecr = new AmazonECRClient(region); + string accountId = await GetAccountIdAsync(ecr); + string registryHost = $"{accountId}.dkr.ecr.{regionName}.amazonaws.com"; + string targetImage = $"{registryHost}/{repository}:{tag}"; + _output.WriteLine($"Target: {targetImage}"); + + await EnsureRepositoryAsync(ecr, repository); + + string? pushedDigest = null; + try + { + string payloadFile = Path.Combine(Path.GetTempPath(), $"oci-itest-{Guid.NewGuid():N}.txt"); + await File.WriteAllTextAsync(payloadFile, "Hello from the AWS.OCI.Builder spill-to-disk integration test.\n"); + + try + { + using OciImageBuilder builder = await OciImageBuilder.FromBaseImageAsync(baseImage, Platform); + await builder.AddLayerAsync( + new FileManifest { { payloadFile, "/app/hello.txt" } }, + createdBy: "AWS.OCI.Builder spill-to-disk integration test"); + builder.Config.SetWorkingDirectory("/app"); + + // Force the spill-to-disk copy path: a 1 MiB threshold is well below the base image's real layer + // sizes (its largest runtime-deps layer is tens of MiB), so those layers are streamed through a + // temp file, re-verified from disk, and uploaded from the file rather than buffered in memory. + var options = new EcrPushOptions { MaxInMemoryLayerBytes = 1L * 1024 * 1024 }; + string authToken = await GetAuthorizationTokenStringAsync(ecr); + pushedDigest = await builder.PushToEcrAsync(targetImage, authToken, options); + } + finally + { + File.Delete(payloadFile); + } + + _output.WriteLine($"Pushed digest (spill path): {pushedDigest}"); + Assert.StartsWith("sha256:", pushedDigest); + + // Verify via ECR that the tag exists and the digest matches — proving the disk-spilled layers produced + // a valid, consistent image in ECR (every blob committed its digest check). + DescribeImagesResponse described = await ecr.DescribeImagesAsync(new DescribeImagesRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + + ImageDetail detail = Assert.Single(described.ImageDetails); + Assert.Contains(tag, detail.ImageTags); + Assert.Equal(pushedDigest, detail.ImageDigest); + _output.WriteLine($"Verified spill-path image {detail.ImageDigest} ({detail.ImageSizeInBytes} bytes) in ECR."); + } + finally + { + try + { + await ecr.BatchDeleteImageAsync(new BatchDeleteImageRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + _output.WriteLine($"Cleaned up tag {tag}."); + } + catch (Exception ex) + { + _output.WriteLine($"Cleanup of tag {tag} failed (non-fatal): {ex.Message}"); + } + } + } + + [Fact] + public async Task Pushed_executable_file_keeps_its_0755_mode_in_ecr() + { + string repository = EnvOr("OCI_ECR_REPOSITORY", DefaultRepository); + string regionName = EnvOr("OCI_ECR_REGION", DefaultRegion); + string baseImage = EnvOr("OCI_ECR_BASE_IMAGE", DefaultBaseImage); + RegionEndpoint region = RegionEndpoint.GetBySystemName(regionName); + string tag = "itest-exec-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss") + "-" + Guid.NewGuid().ToString("N")[..6]; + + using var ecr = new AmazonECRClient(region); + string accountId = await GetAccountIdAsync(ecr); + string registryHost = $"{accountId}.dkr.ecr.{regionName}.amazonaws.com"; + string targetImage = $"{registryHost}/{repository}:{tag}"; + await EnsureRepositoryAsync(ecr, repository); + + string? newLayerDigest = null; + try + { + string scriptFile = Path.Combine(Path.GetTempPath(), $"oci-itest-{Guid.NewGuid():N}.sh"); + await File.WriteAllTextAsync(scriptFile, "#!/bin/sh\necho hi\n"); + try + { + using OciImageBuilder builder = await OciImageBuilder.FromBaseImageAsync(baseImage, Platform); + // Mark the script executable regardless of host — this is the behavior that must survive to ECR, + // and on Windows there would be no source executable bit to rely on. + var files = new FileManifest(); + files.AddExecutable(scriptFile, "/app/run.sh"); + await builder.AddLayerAsync(files, createdBy: "exec-bit itest"); + string authToken = await GetAuthorizationTokenStringAsync(ecr); + await builder.PushToEcrAsync(targetImage, authToken); + } + finally + { + File.Delete(scriptFile); + } + + // Pull the pushed manifest, find the LAST layer (the one we added), download its blob, and inspect the + // mode of our file entry inside the gzipped tar. + var auth = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); + string basic = auth.AuthorizationData[0].AuthorizationToken; + + using var http = new HttpClient(); + http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", basic); + foreach (string accept in new[] + { + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + }) + { + http.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue(accept)); + } + + string manifestJson = await http.GetStringAsync($"https://{registryHost}/v2/{repository}/manifests/{tag}"); + using var manifestDoc = System.Text.Json.JsonDocument.Parse(manifestJson); + var layers = manifestDoc.RootElement.GetProperty("layers"); + newLayerDigest = layers[layers.GetArrayLength() - 1].GetProperty("digest").GetString(); + _output.WriteLine($"New layer blob: {newLayerDigest}"); + + byte[] gzipped = await http.GetByteArrayAsync($"https://{registryHost}/v2/{repository}/blobs/{newLayerDigest}"); + UnixFileMode mode = ReadEntryMode(gzipped, "app/run.sh"); + _output.WriteLine($"app/run.sh mode in ECR: {mode}"); + + Assert.True(mode.HasFlag(UnixFileMode.UserExecute), $"expected the owner-execute bit to survive, got {mode}"); + } + finally + { + try + { + await ecr.BatchDeleteImageAsync(new BatchDeleteImageRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + } + catch (Exception ex) { _output.WriteLine($"Cleanup failed (non-fatal): {ex.Message}"); } + } + } + + /// Decompresses a gzipped tar blob and returns the Unix mode of the entry named . + private static UnixFileMode ReadEntryMode(byte[] gzippedTar, string entryName) + { + using var input = new MemoryStream(gzippedTar); + using var gzip = new System.IO.Compression.GZipStream(input, System.IO.Compression.CompressionMode.Decompress); + using var reader = new System.Formats.Tar.TarReader(gzip); + while (reader.GetNextEntry() is { } entry) + { + if (entry.Name.TrimStart('\0') == entryName) + { + return entry.Mode; + } + } + throw new Xunit.Sdk.XunitException($"Entry '{entryName}' not found in the pulled layer."); + } + + private static async Task GetAccountIdAsync(IAmazonECR ecr) + { + // GetAuthorizationToken's response identifies the registry (account) the caller is authenticated against. + GetAuthorizationTokenResponse token = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); + AuthorizationData data = Assert.Single(token.AuthorizationData); + + // proxyEndpoint is https://.dkr.ecr..amazonaws.com — extract the account id from the host. + var proxy = new Uri(data.ProxyEndpoint); + return proxy.Host.Split('.')[0]; + } + + /// Fetches the ECR authorization token string (base64 "AWS:password") to pass to PushToEcrAsync. + private static async Task GetAuthorizationTokenStringAsync(IAmazonECR ecr) + { + GetAuthorizationTokenResponse token = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); + AuthorizationData data = Assert.Single(token.AuthorizationData); + return data.AuthorizationToken; + } + + private async Task EnsureRepositoryAsync(IAmazonECR ecr, string repository) + { + try + { + await ecr.CreateRepositoryAsync(new CreateRepositoryRequest { RepositoryName = repository }); + _output.WriteLine($"Created repository {repository}."); + } + catch (RepositoryAlreadyExistsException) + { + _output.WriteLine($"Repository {repository} already exists."); + } + } + + private static string EnvOr(string name, string fallback) + { + string? value = Environment.GetEnvironmentVariable(name); + return string.IsNullOrWhiteSpace(value) ? fallback : value; + } + + /// + /// A deterministic that invokes the callback synchronously on the reporting thread, + /// so all events are delivered by the time the awaited push completes — unlike , whose + /// asynchronous marshaling would make post-push assertions race the final callbacks. + /// + private sealed class SynchronousProgress : IProgress + { + private readonly Action _onReport; + public SynchronousProgress(Action onReport) => _onReport = onReport; + public void Report(LayerPushProgress value) => _onReport(value); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj b/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj new file mode 100644 index 000000000..2578b3bb6 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj @@ -0,0 +1,27 @@ + + + + + + $(DefaultPackageTargets) + false + + false + false + enable + enable + + + + + + + + + + + + + + + diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs new file mode 100644 index 000000000..d3d9f615a --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs @@ -0,0 +1,444 @@ +using System.Net; +using System.Net.Http.Headers; +using AWS.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class AuthHandshakeHandlerTests +{ + /// A programmable inner handler that responds based on the request URI. + private sealed class StubHandler : HttpMessageHandler + { + private readonly Func _responder; + public List Requests { get; } = new(); + + public StubHandler(Func responder) => _responder = responder; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.FromResult(_responder(request)); + } + } + + private sealed class StaticCredentials : IRegistryCredentialSource + { + private readonly BasicRegistryCredentials? _creds; + public StaticCredentials(BasicRegistryCredentials? creds) => _creds = creds; + public Task GetCredentialsAsync(string registryHost, CancellationToken ct = default) + => Task.FromResult(_creds); + } + + [Fact] + public async Task Performs_bearer_handshake_and_retries_with_token() + { + var stub = new StubHandler(request => + { + string url = request.RequestUri!.ToString(); + + // The token endpoint: return a token when Basic auth was presented. + if (url.StartsWith("https://auth.example.com/token")) + { + Assert.Equal("Basic", request.Headers.Authorization?.Scheme); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + // The resource endpoint: 401 until a Bearer token is presented. + if (request.Headers.Authorization?.Scheme == "Bearer") + { + Assert.Equal("abc123", request.Headers.Authorization.Parameter); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("ok") }; + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", + "realm=\"https://auth.example.com/token\",service=\"registry.example.com\",scope=\"repository:app:pull\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/app/manifests/latest"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("ok", await response.Content.ReadAsStringAsync()); + + // Expect: initial 401, token GET, authenticated retry. + Assert.Equal(3, stub.Requests.Count); + } + + [Fact] + public async Task Selects_bearer_challenge_when_it_is_not_the_first_challenge() + { + // A registry may return multiple WWW-Authenticate challenges in any order. The handler must scan all of + // them and pick the usable Bearer challenge even when an unsupported scheme is listed first — the previous + // implementation inspected only the first challenge and would have failed here. + var stub = new StubHandler(request => + { + string url = request.RequestUri!.ToString(); + + if (url.StartsWith("https://auth.example.com/token")) + { + Assert.Equal("Basic", request.Headers.Authorization?.Scheme); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + Assert.Equal("abc123", request.Headers.Authorization.Parameter); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("ok") }; + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + // Unsupported scheme first, usable Bearer second. + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue("Negotiate")); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", + "realm=\"https://auth.example.com/token\",service=\"registry.example.com\",scope=\"repository:app:pull\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/app/manifests/latest"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("ok", await response.Content.ReadAsStringAsync()); + Assert.Equal(3, stub.Requests.Count); + } + + [Fact] + public async Task Falls_back_to_basic_when_no_usable_bearer_challenge_is_present() + { + // When only an unsupported scheme and a Basic challenge are offered, the handler should select Basic and + // present Basic credentials directly (no token round-trip). + var stub = new StubHandler(request => + { + if (request.Headers.Authorization?.Scheme == "Basic") + { + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("ok") }; + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue("Negotiate")); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue("Basic", "realm=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/app/manifests/latest"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("ok", await response.Content.ReadAsStringAsync()); + // Initial 401 + authenticated (Basic) retry — no token endpoint call. + Assert.Equal(2, stub.Requests.Count); + } + + [Fact] + public async Task Rejects_bearer_realm_pointing_at_loopback() + { + var stub = new StubHandler(_ => + { + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://127.0.0.1/token\",service=\"evil\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + await Assert.ThrowsAsync( + () => client.GetAsync("https://registry.example.com/v2/app/manifests/latest")); + } + + [Fact] + public async Task Passes_through_non_401_responses() + { + var stub = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("direct") }); + var handler = new AuthHandshakeHandler(new AnonymousRegistryCredentialSource(), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Single(stub.Requests); + } + + private static readonly HttpRequestOptionsKey MarkerOption = new("test-marker"); + + [Theory] + // ECR / Docker Hub style (no whitespace). + [InlineData("realm=\"https://a/token\",service=\"reg\",scope=\"repository:app:pull\"", "https://a/token", "reg", "repository:app:pull")] + // Optional whitespace after commas. + [InlineData("realm=\"https://a/token\", service=\"reg\", scope=\"repository:app:pull\"", "https://a/token", "reg", "repository:app:pull")] + // Optional whitespace BEFORE commas — the old regex silently dropped realm here. + [InlineData("realm=\"https://a/token\" , service=\"reg\"", "https://a/token", "reg", null)] + // Whitespace around '=' and tabs. + [InlineData("realm = \"https://a/token\",\tservice\t=\t\"reg\"", "https://a/token", "reg", null)] + // Unquoted token values — the old regex parsed nothing. + [InlineData("realm=https://a/token,service=reg", "https://a/token", "reg", null)] + // A comma inside a quoted value stays part of the value. + [InlineData("realm=\"https://a/token\",scope=\"repository:app:pull,push\"", "https://a/token", null, "repository:app:pull,push")] + public void Parses_bearer_challenge_parameters(string parameters, string realm, string? service, string? scope) + { + Dictionary parsed = AuthHandshakeHandler.ParseBearerParameters(parameters); + + Assert.Equal(realm, parsed["realm"]); + Assert.Equal(service, parsed.GetValueOrDefault("service")); + Assert.Equal(scope, parsed.GetValueOrDefault("scope")); + } + + [Fact] + public void Parses_hyphenated_names_escaped_quotes_and_ignores_malformed_segments() + { + Dictionary parsed = AuthHandshakeHandler.ParseBearerParameters( + "realm=\"https://a/token\", garbage, x-custom=\"1\", error_description=\"say \\\"hi\\\"\", REALM=\"second\""); + + Assert.Equal("https://a/token", parsed["realm"]); // first occurrence wins; names are case-insensitive + Assert.Equal("1", parsed["x-custom"]); // hyphen is a valid token character + Assert.Equal("say \"hi\"", parsed["error_description"]); // backslash escapes are unescaped + Assert.False(parsed.ContainsKey("garbage")); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Parses_empty_parameters_as_empty(string? parameters) + => Assert.Empty(AuthHandshakeHandler.ParseBearerParameters(parameters)); + + [Fact] + public async Task Handshake_succeeds_with_whitespace_before_commas_in_challenge() + { + var stub = new StubHandler(request => + { + if (request.RequestUri!.ToString().StartsWith("https://auth.example.com/token")) + { + Assert.Contains("service=registry.example.com", request.RequestUri.Query, StringComparison.Ordinal); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + return new HttpResponseMessage(HttpStatusCode.OK); + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\" , service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(3, stub.Requests.Count); + } + + [Fact] + public async Task Handshake_retry_is_disposed_without_closing_the_callers_body_stream() + { + var stub = new StubHandler(request => + { + if (request.RequestUri!.ToString().StartsWith("https://auth.example.com/token")) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + return new HttpResponseMessage(HttpStatusCode.Accepted); + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\",service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + // A seekable body, as in a real layer upload (FileStream / MemoryStream) — owned by the caller. + var body = new MemoryStream("layer-bytes"u8.ToArray()); + using var request = new HttpRequestMessage(HttpMethod.Patch, "https://registry.example.com/v2/app/blobs/uploads/1") + { + Content = new StreamContent(body), + }; + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + + // The retry clone was disposed: its content can no longer be read. + HttpRequestMessage retry = stub.Requests[^1]; + Assert.NotSame(request, retry); + await Assert.ThrowsAsync(() => retry.Content!.ReadAsByteArrayAsync()); + + // ...but the caller's stream is still open and usable. + Assert.True(body.CanRead); + body.Position = 0; + Assert.Equal("layer-bytes"u8.ToArray(), body.ToArray()); + } + + [Fact] + public async Task Handshake_retry_preserves_version_policy_and_request_options() + { + var stub = new StubHandler(request => + { + string url = request.RequestUri!.ToString(); + if (url.StartsWith("https://auth.example.com/token")) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + return new HttpResponseMessage(HttpStatusCode.OK); + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\",service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using var request = new HttpRequestMessage(HttpMethod.Get, "https://registry.example.com/v2/"); + request.VersionPolicy = HttpVersionPolicy.RequestVersionExact; + request.Options.Set(MarkerOption, "kept"); + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + // The last request is the authenticated retry produced by CloneAsync. + HttpRequestMessage retry = stub.Requests[^1]; + Assert.NotSame(request, retry); + Assert.Equal(HttpVersionPolicy.RequestVersionExact, retry.VersionPolicy); + Assert.True(retry.Options.TryGetValue(MarkerOption, out string? marker)); + Assert.Equal("kept", marker); + } + + [Fact] + public async Task Caller_supplied_authorization_is_not_overwritten_by_cached_auth() + { + var stub = new StubHandler(request => + { + string url = request.RequestUri!.ToString(); + if (url.StartsWith("https://auth.example.com/token")) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + // Echo back which token the registry received. + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(request.Headers.Authorization.Parameter!) }; + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\",service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + // 1. Prime the cache with a handshake-derived token for the host. + using (HttpResponseMessage primed = await client.GetAsync("https://registry.example.com/v2/")) + { + Assert.Equal("abc123", await primed.Content.ReadAsStringAsync()); + } + + // 2. A request carrying the caller's own Authorization must reach the registry unchanged. + using var request = new HttpRequestMessage(HttpMethod.Get, "https://registry.example.com/v2/app/manifests/latest"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "caller-token"); + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal("caller-token", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task Caller_supplied_authorization_401_is_returned_without_a_handshake() + { + var stub = new StubHandler(_ => + { + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\",service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using var request = new HttpRequestMessage(HttpMethod.Get, "https://registry.example.com/v2/"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "caller-token"); + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + Assert.Single(stub.Requests); // no token request, no retry + } + + [Fact] + public async Task Requests_marked_skip_authentication_are_not_authenticated() + { + var stub = new StubHandler(_ => + { + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue("Basic", "realm=\"x\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using var request = new HttpRequestMessage(HttpMethod.Get, "https://storage.example.com/blob"); + request.Options.Set(AuthHandshakeHandler.SkipAuthentication, true); + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + Assert.Single(stub.Requests); + Assert.Null(stub.Requests[0].Headers.Authorization); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs new file mode 100644 index 000000000..1959511d6 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs @@ -0,0 +1,99 @@ +using System.Net; +using System.Net.Http.Headers; +using AWS.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +/// +/// Verifies that following a blob-download redirect to a different host never presents registry credentials to +/// the redirect target, even when that target answers with a 401 authentication challenge. +/// +public class BlobRedirectAuthTests +{ + private const string RegistryHost = "registry.example.com"; + private const string Digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + + private sealed class StaticCredentials : IRegistryCredentialSource + { + public Task GetCredentialsAsync(string registryHost, CancellationToken ct = default) + => Task.FromResult(new BasicRegistryCredentials("AWS", "secret")); + } + + /// Records every request and delegates the response to a callback. + private sealed class RecordingHandler : HttpMessageHandler + { + private readonly Func _respond; + public List<(Uri Uri, AuthenticationHeaderValue? Auth)> Requests { get; } = new(); + + public RecordingHandler(Func respond) => _respond = respond; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add((request.RequestUri!, request.Headers.Authorization)); + return Task.FromResult(_respond(request)); + } + } + + private static HttpResponseMessage RedirectTo(string location) + { + var redirect = new HttpResponseMessage(HttpStatusCode.TemporaryRedirect); + redirect.Headers.Location = new Uri(location); + return redirect; + } + + [Fact] + public async Task Cross_host_redirect_target_challenge_does_not_receive_credentials() + { + var stub = new RecordingHandler(request => + { + Uri uri = request.RequestUri!; + if (uri.Host == RegistryHost) + { + // The registry redirects the blob download to a host it controls. + return RedirectTo("https://attacker.example.com/blob"); + } + + // The redirect target tries to harvest credentials with a challenge (Bearer realm on its own host). + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://attacker.example.com/token\",service=\"attacker\"")); + return unauthorized; + }); + + var client = new RegistryClient( + RegistryHost, new HttpClient(new AuthHandshakeHandler(new StaticCredentials(), stub))); + + await Assert.ThrowsAnyAsync(() => client.GetBlobAsync("app", Digest)); + + // The attacker host was contacted exactly once (the redirect hop), with no Authorization header, and its + // token endpoint was never called. + var attackerRequests = stub.Requests.Where(r => r.Uri.Host == "attacker.example.com").ToList(); + Assert.Single(attackerRequests); + Assert.Null(attackerRequests[0].Auth); + Assert.DoesNotContain(stub.Requests, r => r.Uri.AbsolutePath == "/token"); + } + + [Fact] + public async Task Cross_host_redirect_to_storage_succeeds_without_credentials() + { + byte[] payload = "layer-bytes"u8.ToArray(); + var stub = new RecordingHandler(request => + request.RequestUri!.Host == RegistryHost + ? RedirectTo("https://storage.example.com/presigned?sig=abc") + : new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(payload) }); + + var client = new RegistryClient( + RegistryHost, new HttpClient(new AuthHandshakeHandler(new StaticCredentials(), stub))); + + await using (Stream stream = await client.GetBlobAsync("app", Digest)) + { + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer); + Assert.Equal(payload, buffer.ToArray()); + } + + var storageRequest = Assert.Single(stub.Requests, r => r.Uri.Host == "storage.example.com"); + Assert.Null(storageRequest.Auth); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs new file mode 100644 index 000000000..d3f19f4f5 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs @@ -0,0 +1,192 @@ +using System.Text; +using AWS.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class DigestVerificationTests +{ + [Fact] + public void ComputeDigest_of_known_bytes() + { + // SHA-256 of the empty input is a well-known constant. + string digest = RegistryClient.ComputeDigest(ReadOnlySpan.Empty); + Assert.Equal("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", digest); + } + + [Fact] + public void ComputeDigest_byte_and_stream_overloads_agree() + { + byte[] data = Encoding.UTF8.GetBytes("the quick brown fox"); + string fromBytes = RegistryClient.ComputeDigest(data); + string fromStream = RegistryClient.ComputeDigest(new MemoryStream(data)); + Assert.Equal(fromStream, fromBytes); + } + + [Fact] + public void ComputeDigest_stream_hashes_from_current_position() + { + byte[] data = Encoding.UTF8.GetBytes("prefix|suffix"); + var stream = new MemoryStream(data) { Position = 7 }; // positioned at "suffix" + + string digest = RegistryClient.ComputeDigest(stream); + + Assert.Equal(RegistryClient.ComputeDigest(Encoding.UTF8.GetBytes("suffix")), digest); + Assert.Equal(data.Length, stream.Position); // left at the end, not rewound + } + + [Fact] + public void VerifyDigest_passes_for_matching_content() + { + byte[] data = Encoding.UTF8.GetBytes("layer-bytes"); + string digest = RegistryClient.ComputeDigest(data); + + // Should not throw. + RegistryClient.VerifyDigest("test blob", digest, data); + } + + [Fact] + public void VerifyDigest_is_case_insensitive_on_hex() + { + byte[] data = Encoding.UTF8.GetBytes("layer-bytes"); + string digest = RegistryClient.ComputeDigest(data).ToUpperInvariant(); // SHA256: + RegistryClient.VerifyDigest("test blob", digest, data); + } + + [Fact] + public void VerifyDigest_throws_on_mismatch() + { + byte[] data = Encoding.UTF8.GetBytes("real-bytes"); + byte[] other = Encoding.UTF8.GetBytes("tampered-bytes"); + string wrongDigest = RegistryClient.ComputeDigest(other); + + DigestMismatchException ex = Assert.Throws( + () => RegistryClient.VerifyDigest("test blob", wrongDigest, data)); + + Assert.Equal(wrongDigest, ex.ExpectedDigest); + Assert.Equal(RegistryClient.ComputeDigest(data), ex.ActualDigest); + } + + [Fact] + public void VerifyDigest_rejects_unsupported_algorithm() + { + byte[] data = Encoding.UTF8.GetBytes("x"); + RegistryException ex = Assert.Throws( + () => RegistryClient.VerifyDigest("test blob", "sha512:deadbeef", data)); + Assert.Contains("sha256", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void DigestMismatchException_is_a_RegistryException() + => Assert.IsAssignableFrom(new DigestMismatchException("x", "sha256:a", "sha256:b")); + + /// A stub registry handler that serves a single blob's bytes under a given digest on GET. + private sealed class SingleBlobHandler : HttpMessageHandler + { + private readonly string _digest; + private readonly byte[] _bytes; + + public SingleBlobHandler(string digest, byte[] bytes) + { + _digest = digest; + _bytes = bytes; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.Method == HttpMethod.Get && request.RequestUri!.AbsolutePath.EndsWith($"/blobs/{_digest}", StringComparison.Ordinal)) + { + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new ByteArrayContent(_bytes), + }); + } + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.NotFound)); + } + } + + [Fact] + public async Task GetVerifiedBlobToFileAsync_returns_verified_stream_and_deletes_file_on_dispose() + { + byte[] data = Encoding.UTF8.GetBytes("a large-ish base layer spilled to disk"); + string digest = RegistryClient.ComputeDigest(data); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(digest, data))); + + string path; + await using (VerifiedBlobFile handle = await client.GetVerifiedBlobToFileAsync("repo", digest, "base layer")) + { + path = handle.Path; + Assert.True(File.Exists(path)); + Assert.Equal(0, handle.Stream.Position); + + // The stream serves exactly the verified bytes. + using var buffer = new MemoryStream(); + await handle.Stream.CopyToAsync(buffer); + Assert.Equal(data, buffer.ToArray()); + } + + // Disposing the handle deletes the temp file. + Assert.False(File.Exists(path)); + } + + [Fact] + public async Task GetVerifiedBlobToFileAsync_throws_when_bytes_do_not_match_digest() + { + byte[] served = Encoding.UTF8.GetBytes("corrupted-on-the-wire"); + // Declare a digest for DIFFERENT content, so the download hash will not match. + string declaredDigest = RegistryClient.ComputeDigest(Encoding.UTF8.GetBytes("the-real-content")); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(declaredDigest, served))); + + await Assert.ThrowsAsync( + () => client.GetVerifiedBlobToFileAsync("repo", declaredDigest, "base layer")); + // Cleanup-on-failure is covered deterministically by the success test's dispose assertion; a global + // temp-dir file count here would race with other parallel tests sharing the temp directory. + } + + [Fact] + public async Task GetVerifiedBlobToFileAsync_throws_when_declared_size_differs_from_blob_length() + { + // The bytes hash correctly to the digest, but the manifest descriptor declares the wrong size. + byte[] data = Encoding.UTF8.GetBytes("correct content, wrong declared size"); + string digest = RegistryClient.ComputeDigest(data); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(digest, data))); + + BlobSizeMismatchException ex = await Assert.ThrowsAsync( + () => client.GetVerifiedBlobToFileAsync("repo", digest, "base layer", expectedSize: data.Length + 7)); + + Assert.Equal(digest, ex.Digest); + Assert.Equal(data.Length + 7, ex.DeclaredSize); + Assert.Equal(data.Length, ex.ActualSize); + } + + [Fact] + public async Task GetVerifiedBlobToFileAsync_accepts_matching_declared_size() + { + byte[] data = Encoding.UTF8.GetBytes("size matches"); + string digest = RegistryClient.ComputeDigest(data); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(digest, data))); + + await using VerifiedBlobFile handle = await client.GetVerifiedBlobToFileAsync("repo", digest, "base layer", expectedSize: data.Length); + Assert.Equal(data.Length, handle.Stream.Length); + } + + [Fact] + public async Task GetVerifiedBlobBytesAsync_throws_when_declared_size_differs_from_blob_length() + { + byte[] data = Encoding.UTF8.GetBytes("small blob"); + string digest = RegistryClient.ComputeDigest(data); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(digest, data))); + + BlobSizeMismatchException ex = await Assert.ThrowsAsync( + () => client.GetVerifiedBlobBytesAsync("repo", digest, "base image config", expectedSize: data.Length - 1)); + Assert.Contains("declares a size", ex.Message, StringComparison.Ordinal); + + // A matching size (or no declared size) passes. + Assert.Equal(data, await client.GetVerifiedBlobBytesAsync("repo", digest, "base image config", expectedSize: data.Length)); + Assert.Equal(data, await client.GetVerifiedBlobBytesAsync("repo", digest, "base image config")); + } + + [Fact] + public void BlobSizeMismatchException_is_a_RegistryException() + => Assert.IsAssignableFrom(new BlobSizeMismatchException("x", "sha256:a", 1, 2)); +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs new file mode 100644 index 000000000..d76baeea6 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs @@ -0,0 +1,21 @@ +using AWS.OCI.Builder; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class FileManifestTests +{ + [Fact] + public void Collection_initializer_adds_entries_in_order() + { + var files = new FileManifest + { + { "/local/app.dll", "/app/app.dll" }, + { "/local/config", "/app/config" }, + }; + + Assert.Equal(2, files.Entries.Count); + Assert.Equal("/local/app.dll", files.Entries[0].LocalPath); + Assert.Equal("/app/config", files.Entries[1].ContainerPath); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs new file mode 100644 index 000000000..1f4aa1561 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs @@ -0,0 +1,269 @@ +using System.Security.Cryptography; +using System.Text.Json.Nodes; +using AWS.OCI.Builder; +using AWS.OCI.Builder.Images; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class ImageConfigTests +{ + private const string MinimalConfig = """ + {"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:aaa"]},"history":[{"created":"2020-01-01T00:00:00Z"}],"config":{"Env":["PATH=/usr/bin"]}} + """; + + [Fact] + public void Parse_exposes_os_and_architecture() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + Assert.Equal("linux", config.Os); + Assert.Equal("amd64", config.Architecture); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("[1,2,3]")] // not an object + [InlineData("{ not json")] + public void Parse_rejects_invalid_input(string json) + => Assert.Throws(() => ImageConfig.Parse(json)); + + [Fact] + public void AddLayer_appends_diffId_and_history_entry() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + config.AddLayer("sha256:bbb", "ADD app layer"); + + JsonObject root = (JsonObject)JsonNode.Parse(config.ToJsonString())!; + JsonArray diffIds = (JsonArray)root["rootfs"]!["diff_ids"]!; + JsonArray history = (JsonArray)root["history"]!; + + Assert.Equal(2, diffIds.Count); + Assert.Equal("sha256:bbb", diffIds[1]!.GetValue()); + Assert.Equal(2, history.Count); + Assert.Equal("ADD app layer", history[1]!["created_by"]!.GetValue()); + } + + [Fact] + public void SetEnvironmentVariable_replaces_existing_key_in_place() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + config.SetEnvironmentVariable("PATH", "/custom/bin"); // overwrite existing + config.SetEnvironmentVariable("ASPNETCORE_URLS", "http://+:8080"); // add new + + JsonObject root = (JsonObject)JsonNode.Parse(config.ToJsonString())!; + JsonArray env = (JsonArray)root["config"]!["Env"]!; + + Assert.Equal(2, env.Count); + Assert.Contains("PATH=/custom/bin", env.Select(e => e!.GetValue())); + Assert.DoesNotContain("PATH=/usr/bin", env.Select(e => e!.GetValue())); + Assert.Contains("ASPNETCORE_URLS=http://+:8080", env.Select(e => e!.GetValue())); + } + + [Fact] + public void Sets_entrypoint_cmd_workdir_user_labels_and_ports() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + config.SetEntrypoint("dotnet", "/app/app.dll"); + config.SetCmd("--urls", "http://+:80"); + config.SetWorkingDirectory("/app"); + config.SetUser("1000"); + config.SetLabel("org.opencontainers.image.source", "https://example.com/repo"); + config.ExposePort("8080"); + + JsonObject c = (JsonObject)JsonNode.Parse(config.ToJsonString())!["config"]!; + + Assert.Equal(new[] { "dotnet", "/app/app.dll" }, ((JsonArray)c["Entrypoint"]!).Select(e => e!.GetValue())); + Assert.Equal(new[] { "--urls", "http://+:80" }, ((JsonArray)c["Cmd"]!).Select(e => e!.GetValue())); + Assert.Equal("/app", c["WorkingDir"]!.GetValue()); + Assert.Equal("1000", c["User"]!.GetValue()); + Assert.Equal("https://example.com/repo", c["Labels"]!["org.opencontainers.image.source"]!.GetValue()); + Assert.NotNull(c["ExposedPorts"]!["8080/tcp"]); // bare port got /tcp suffix + } + + [Fact] + public void Serialization_is_compact() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + string json = config.ToJsonString(); + Assert.DoesNotContain("\n", json); + Assert.DoesNotContain(" ", json); + } +} + +public class ManifestTests +{ + [Fact] + public void Round_trips_config_and_layers() + { + var manifest = new Manifest + { + MediaType = MediaTypes.OciManifestV1, + Config = new Descriptor { MediaType = MediaTypes.OciImageConfigV1, Size = 10, Digest = "sha256:cfg" }, + Layers = + { + new Descriptor { MediaType = MediaTypes.OciLayerGzipV1, Size = 100, Digest = "sha256:l1" }, + }, + }; + + Manifest parsed = Manifest.Parse(manifest.ToJsonString()); + + Assert.Equal(2, parsed.SchemaVersion); + Assert.Equal(MediaTypes.OciManifestV1, parsed.MediaType); + Assert.Equal("sha256:cfg", parsed.Config.Digest); + Assert.Single(parsed.Layers); + Assert.Equal("sha256:l1", parsed.Layers[0].Digest); + } + + [Fact] + public void Property_order_is_schemaVersion_mediaType_config_layers() + { + var manifest = new Manifest + { + MediaType = MediaTypes.OciManifestV1, + Config = new Descriptor { MediaType = MediaTypes.OciImageConfigV1, Size = 1, Digest = "sha256:cfg" }, + }; + string json = manifest.ToJsonString(); + + int iSchema = json.IndexOf("schemaVersion", StringComparison.Ordinal); + int iMedia = json.IndexOf("mediaType", StringComparison.Ordinal); + int iConfig = json.IndexOf("\"config\"", StringComparison.Ordinal); + int iLayers = json.IndexOf("\"layers\"", StringComparison.Ordinal); + + Assert.True(iSchema < iMedia && iMedia < iConfig && iConfig < iLayers); + } +} + +public class ImageIndexTests +{ + private const string IndexJson = """ + {"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[ + {"mediaType":"application/vnd.oci.image.manifest.v1+json","size":1,"digest":"sha256:amd","platform":{"os":"linux","architecture":"amd64"}}, + {"mediaType":"application/vnd.oci.image.manifest.v1+json","size":1,"digest":"sha256:arm","platform":{"os":"linux","architecture":"arm64"}}, + {"mediaType":"application/vnd.oci.image.manifest.v1+json","size":1,"digest":"sha256:att","platform":{"os":"unknown","architecture":"unknown"}} + ]} + """; + + [Fact] + public void FindManifest_selects_matching_platform() + { + ImageIndex index = ImageIndex.Parse(IndexJson); + PlatformManifestDescriptor? amd = index.FindManifest("linux", "amd64"); + PlatformManifestDescriptor? arm = index.FindManifest("linux", "arm64"); + + Assert.Equal("sha256:amd", amd!.Digest); + Assert.Equal("sha256:arm", arm!.Digest); + } + + [Fact] + public void FindManifest_returns_null_for_absent_platform() + { + ImageIndex index = ImageIndex.Parse(IndexJson); + Assert.Null(index.FindManifest("windows", "amd64")); + } +} + +public class ImageBuilderTests +{ + private const string BaseConfig = """ + {"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:base1"]},"history":[{"created":"2020-01-01T00:00:00Z"}]} + """; + + private static string BaseManifest(string layerMediaType, string configMediaType) => $$""" + {"schemaVersion":2,"mediaType":"{{MediaTypes.OciManifestV1}}","config":{"mediaType":"{{configMediaType}}","size":10,"digest":"sha256:basecfg"},"layers":[{"mediaType":"{{layerMediaType}}","size":100,"digest":"sha256:baselayer"}]} + """; + + private static Layer FakeLayer(string mediaType) + { + // Build a real tiny layer so Digest/DiffId are genuine sha256 values. The layer is fully built (into a + // throwaway stream) before returning, so its source directory can be removed immediately. + string tmp = Path.Combine(Path.GetTempPath(), "aws-oci-ib-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tmp); + try + { + string file = Path.Combine(tmp, "app.dll"); + File.WriteAllText(file, "payload"); + var manifest = new FileManifest { { file, "/app/app.dll" } }; + return Layer.CreateAsync(manifest, new MemoryStream(), mediaType).GetAwaiter().GetResult(); + } + finally + { + Directory.Delete(tmp, recursive: true); + } + } + + [Fact] + public void AddLayer_keeps_manifest_layers_and_config_diffids_in_lockstep() + { + ImageBuilder builder = ImageBuilder.FromBaseImage( + BaseConfig, + BaseManifest(MediaTypes.OciLayerGzipV1, MediaTypes.OciImageConfigV1), + MediaTypes.OciManifestV1); + + Layer layer = FakeLayer(MediaTypes.OciLayerGzipV1); + builder.AddLayer(layer, "ADD app"); + + BuiltImage built = builder.Build(); + Manifest manifest = Manifest.Parse(System.Text.Encoding.UTF8.GetString(built.ManifestBytes)); + JsonObject config = (JsonObject)System.Text.Json.Nodes.JsonNode.Parse(System.Text.Encoding.UTF8.GetString(built.ConfigBytes))!; + + // Base had 1 layer / 1 diff_id; after overlay both are 2 and the new entries match the layer. + Assert.Equal(2, manifest.Layers.Count); + Assert.Equal(layer.Digest, manifest.Layers[1].Digest); + + JsonArray diffIds = (JsonArray)config["rootfs"]!["diff_ids"]!; + Assert.Equal(2, diffIds.Count); + Assert.Equal(layer.DiffId, diffIds[1]!.GetValue()); + } + + [Fact] + public void Build_points_manifest_config_descriptor_at_actual_config_bytes() + { + ImageBuilder builder = ImageBuilder.FromBaseImage( + BaseConfig, + BaseManifest(MediaTypes.OciLayerGzipV1, MediaTypes.OciImageConfigV1), + MediaTypes.OciManifestV1); + builder.AddLayer(FakeLayer(MediaTypes.OciLayerGzipV1)); + + BuiltImage built = builder.Build(); + Manifest manifest = Manifest.Parse(System.Text.Encoding.UTF8.GetString(built.ManifestBytes)); + + // The manifest's config descriptor must match the real config bytes (size + digest), or registries reject it. + string expectedDigest = "sha256:" + Convert.ToHexString(SHA256.HashData(built.ConfigBytes)).ToLowerInvariant(); + Assert.Equal(expectedDigest, manifest.Config.Digest); + Assert.Equal(built.ConfigBytes.LongLength, manifest.Config.Size); + Assert.Equal(built.ConfigDigest, manifest.Config.Digest); + } + + [Fact] + public void Detects_oci_schema_and_uses_oci_media_types() + { + ImageBuilder builder = ImageBuilder.FromBaseImage( + BaseConfig, + BaseManifest(MediaTypes.OciLayerGzipV1, MediaTypes.OciImageConfigV1), + MediaTypes.OciManifestV1); + + Assert.Equal(MediaTypes.OciLayerGzipV1, builder.LayerMediaType); + Assert.Equal(MediaTypes.OciImageConfigV1, builder.ConfigMediaType); + Assert.Equal(MediaTypes.OciManifestV1, builder.ManifestMediaType); + } + + [Fact] + public void Detects_docker_schema_from_reported_media_type() + { + ImageBuilder builder = ImageBuilder.FromBaseImage( + BaseConfig, + BaseManifest(MediaTypes.DockerLayerGzip, MediaTypes.DockerImageConfigV1), + MediaTypes.DockerManifestV2); + + Assert.Equal(MediaTypes.DockerLayerGzip, builder.LayerMediaType); + Assert.Equal(MediaTypes.DockerImageConfigV1, builder.ConfigMediaType); + Assert.Equal(MediaTypes.DockerManifestV2, builder.ManifestMediaType); + + // A layer built as OCI is coerced to the Docker layer media type in the manifest. + builder.AddLayer(FakeLayer(MediaTypes.OciLayerGzipV1)); + BuiltImage built = builder.Build(); + Manifest manifest = Manifest.Parse(System.Text.Encoding.UTF8.GetString(built.ManifestBytes)); + Assert.Equal(MediaTypes.DockerLayerGzip, manifest.Layers[^1].MediaType); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs new file mode 100644 index 000000000..164d215e5 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs @@ -0,0 +1,120 @@ +using AWS.OCI.Builder; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class ImageReferenceTests +{ + [Theory] + [InlineData("mcr.microsoft.com/dotnet/aspnet:8.0", "mcr.microsoft.com", "dotnet/aspnet", "8.0", false)] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest", + "123456789012.dkr.ecr.us-west-2.amazonaws.com", "myapp", "latest", false)] + [InlineData("library/ubuntu:22.04", "registry-1.docker.io", "library/ubuntu", "22.04", false)] + public void Parses_explicit_and_hub_references( + string input, string registry, string repository, string reference, bool isDigest) + { + ImageReference parsed = ImageReference.Parse(input); + + Assert.Equal(registry, parsed.Registry); + Assert.Equal(repository, parsed.Repository); + Assert.Equal(reference, parsed.Reference); + Assert.Equal(isDigest, parsed.ReferenceIsDigest); + } + + [Fact] + public void Single_segment_hub_repo_gets_library_prefix_and_latest_tag() + { + ImageReference parsed = ImageReference.Parse("ubuntu"); + + Assert.Equal("registry-1.docker.io", parsed.Registry); + Assert.Equal("library/ubuntu", parsed.Repository); + Assert.Equal("latest", parsed.Reference); + } + + [Fact] + public void Parses_digest_reference() + { + const string digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + ImageReference parsed = ImageReference.Parse($"mcr.microsoft.com/dotnet/runtime@{digest}"); + + Assert.Equal("dotnet/runtime", parsed.Repository); + Assert.Equal(digest, parsed.Reference); + Assert.True(parsed.ReferenceIsDigest); + } + + [Fact] + public void Tag_and_digest_together_strips_the_tag_from_the_repository() + { + // A reference may carry BOTH a tag and a digest; the digest is authoritative and the tag must not leak + // into the repository (which would address the wrong path, e.g. "dotnet/runtime:8.0"). + const string digest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"; + ImageReference parsed = ImageReference.Parse($"mcr.microsoft.com/dotnet/runtime:8.0@{digest}"); + + Assert.Equal("dotnet/runtime", parsed.Repository); + Assert.Equal(digest, parsed.Reference); + Assert.True(parsed.ReferenceIsDigest); + } + + [Theory] + // Empty / doubled path components that could smuggle an alternate authority into request URIs, or uppercase. + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com///attacker.example/app:latest")] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com//app:latest")] + [InlineData("registry.example.com/app//evil:latest")] + [InlineData("registry.example.com/App/Bad:latest")] + public void Rejects_malformed_repository(string input) + => Assert.Throws(() => ImageReference.Parse(input)); + + [Theory] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com/app:latest", "us-west-2")] + [InlineData("123456789012.dkr.ecr.eu-central-1.amazonaws.com/app:latest", "eu-central-1")] + [InlineData("210987654321.dkr.ecr-fips.us-east-1.amazonaws.com/app:latest", "us-east-1")] + public void Detects_ecr_and_extracts_region(string input, string expectedRegion) + { + ImageReference parsed = ImageReference.Parse(input); + + Assert.True(parsed.IsEcr); + Assert.Equal(expectedRegion, parsed.EcrRegion); + } + + [Theory] + [InlineData("mcr.microsoft.com/dotnet/aspnet:8.0")] + [InlineData("registry-1.docker.io/library/ubuntu:22.04")] + public void Non_ecr_references_are_not_ecr(string input) + { + ImageReference parsed = ImageReference.Parse(input); + + Assert.False(parsed.IsEcr); + Assert.Null(parsed.EcrRegion); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Rejects_null_or_blank(string input) + => Assert.Throws(() => ImageReference.Parse(input)); + + [Fact] + public void Preserves_non_default_registry_port() + { + ImageReference parsed = ImageReference.Parse("localhost:5000/myapp:dev"); + + Assert.Equal("localhost:5000", parsed.Registry); + Assert.Equal("myapp", parsed.Repository); + Assert.Equal("dev", parsed.Reference); + } + + [Theory] + [InlineData("localhost/myapp:dev")] + [InlineData("LOCALHOST/myapp:dev")] + [InlineData("LocalHost/myapp:dev")] + public void Detects_localhost_registry_case_insensitively(string input) + { + // "localhost" with no '.' or ':' is only recognized as a registry host via the explicit name check, + // which must be case-insensitive — otherwise "LOCALHOST/myapp" is misread as a Docker Hub repository. + ImageReference parsed = ImageReference.Parse(input); + + Assert.Equal(input.Split('/')[0], parsed.Registry); + Assert.Equal("myapp", parsed.Repository); + Assert.Equal("dev", parsed.Reference); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs new file mode 100644 index 000000000..6d299419d --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs @@ -0,0 +1,215 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using AWS.OCI.Builder; +using AWS.OCI.Builder.Images; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +/// +/// Tests for the streaming layer pipeline: the gzip-header normalizer and hashing pass-through streams, and the +/// destination-stream contract. +/// +public class LayerStreamingTests : IDisposable +{ + private readonly string _tempRoot; + + public LayerStreamingTests() + { + _tempRoot = Path.Combine(Path.GetTempPath(), "aws-oci-builder-stream-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempRoot); + } + + public void Dispose() + { + try { Directory.Delete(_tempRoot, recursive: true); } catch { /* best effort cleanup */ } + } + + private FileManifest SampleManifest() + { + string file = Path.Combine(_tempRoot, "app.dll"); + // Incompressible content so deflate emits several blocks and many writes reach the sink. + var data = new byte[512 * 1024]; + new Random(42).NextBytes(data); + File.WriteAllBytes(file, data); + return new FileManifest { { file, "/app/app.dll" } }; + } + + // ---- GzipHeaderNormalizingStream ------------------------------------------------------------- + + /// A fake "gzip" byte sequence whose header has non-normalized MTIME/OS fields, plus a body. + private static byte[] RawGzipLikeBytes() + { + byte[] bytes = Enumerable.Range(0, 32).Select(i => (byte)(0xA0 + i)).ToArray(); + bytes[0] = 0x1F; bytes[1] = 0x8B; // magic, untouched by the normalizer + return bytes; + } + + private static byte[] Expected(byte[] raw) + { + byte[] expected = (byte[])raw.Clone(); + expected[4] = expected[5] = expected[6] = expected[7] = 0; + expected[9] = 0xFF; + return expected; + } + + public static IEnumerable ChunkPatterns() => new[] + { + new object[] { new[] { 32 } }, // whole buffer in one write + new object[] { Enumerable.Repeat(1, 32).ToArray() }, // one byte at a time + new object[] { new[] { 3, 2, 4, 1, 22 } }, // splits inside MTIME and right before OS + new object[] { new[] { 9, 1, 22 } }, // OS byte written alone + new object[] { new[] { 10, 22 } }, // exactly the header, then the body + new object[] { new[] { 11, 21 } }, // a write spanning the header boundary + }; + + [Theory] + [MemberData(nameof(ChunkPatterns))] + public void Normalizer_patches_header_regardless_of_write_chunking(int[] chunks) + { + byte[] raw = RawGzipLikeBytes(); + byte[] original = (byte[])raw.Clone(); + var sink = new MemoryStream(); + var normalizer = new GzipHeaderNormalizingStream(sink); + + int offset = 0; + foreach (int chunk in chunks) + { + normalizer.Write(raw, offset, chunk); + offset += chunk; + } + + Assert.Equal(Expected(raw), sink.ToArray()); + Assert.Equal(original, raw); // the caller's buffer is never modified + } + + [Theory] + [MemberData(nameof(ChunkPatterns))] + public async Task Normalizer_patches_header_regardless_of_write_chunking_async(int[] chunks) + { + byte[] raw = RawGzipLikeBytes(); + byte[] original = (byte[])raw.Clone(); + var sink = new MemoryStream(); + var normalizer = new GzipHeaderNormalizingStream(sink); + + int offset = 0; + foreach (int chunk in chunks) + { + await normalizer.WriteAsync(raw.AsMemory(offset, chunk)); + offset += chunk; + } + + Assert.Equal(Expected(raw), sink.ToArray()); + Assert.Equal(original, raw); + } + + // ---- HashingWriteStream ----------------------------------------------------------------------- + + [Fact] + public async Task HashingWriteStream_hashes_and_counts_bytes_across_sync_and_async_writes() + { + byte[] data = Enumerable.Range(0, 1000).Select(i => (byte)i).ToArray(); + var sink = new MemoryStream(); + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var stream = new HashingWriteStream(sink, hash); + + stream.Write(data, 0, 100); + stream.Write(data.AsSpan(100, 400)); + await stream.WriteAsync(data.AsMemory(500, 500)); + + Assert.Equal(1000, stream.BytesWritten); + Assert.Equal(data, sink.ToArray()); + Assert.Equal(SHA256.HashData(data), hash.GetHashAndReset()); + } + + // ---- Layer.CreateAsync destination contract --------------------------------------------------- + + [Fact] + public async Task CreateAsync_streams_bytes_matching_the_reported_digests_and_size() + { + var destination = new MemoryStream(); + Layer layer = await Layer.CreateAsync(SampleManifest(), destination); + byte[] blob = destination.ToArray(); + + Assert.Equal(blob.LongLength, layer.Size); + Assert.Equal(RegistryDigest(blob), layer.Digest); + Assert.Equal(RegistryDigest(Gunzip(blob)), layer.DiffId); + } + + [Fact] + public async Task CreateAsync_output_matches_one_shot_compression_of_the_same_tar() + { + // Guards that streaming the tar into the compressor in many small writes produces exactly the bytes the + // previous implementation produced by compressing the whole buffered tar in a single write. + var destination = new MemoryStream(); + await Layer.CreateAsync(SampleManifest(), destination); + byte[] streamed = destination.ToArray(); + + byte[] oneShot = OneShotCompress(Gunzip(streamed)); + + Assert.Equal(oneShot, streamed); + } + + [Fact] + public async Task CreateAsync_normalizes_the_gzip_header() + { + var destination = new MemoryStream(); + await Layer.CreateAsync(SampleManifest(), destination); + byte[] blob = destination.ToArray(); + + Assert.Equal(new byte[] { 0, 0, 0, 0 }, blob[4..8]); // MTIME + Assert.Equal(0xFF, blob[9]); // OS = unknown + } + + [Fact] + public async Task CreateAsync_leaves_destination_open_and_appends_at_its_current_position() + { + byte[] prefix = "existing-content"u8.ToArray(); + var destination = new MemoryStream(); + destination.Write(prefix); + + Layer layer = await Layer.CreateAsync(SampleManifest(), destination); + + Assert.True(destination.CanWrite); // not disposed/closed + byte[] all = destination.ToArray(); + Assert.Equal(prefix, all[..prefix.Length]); + Assert.Equal(prefix.Length + layer.Size, all.LongLength); + Assert.Equal(layer.Digest, RegistryDigest(all[prefix.Length..])); + } + + [Fact] + public async Task CreateAsync_validates_the_destination() + { + await Assert.ThrowsAsync(() => Layer.CreateAsync(SampleManifest(), null!)); + + using var readOnly = new MemoryStream(new byte[16], writable: false); + await Assert.ThrowsAsync(() => Layer.CreateAsync(SampleManifest(), readOnly)); + } + + // ---- helpers ---------------------------------------------------------------------------------- + + private static string RegistryDigest(byte[] data) + => "sha256:" + Convert.ToHexString(SHA256.HashData(data)).ToLowerInvariant(); + + private static byte[] Gunzip(byte[] gz) + { + using var gunzip = new GZipStream(new MemoryStream(gz), CompressionMode.Decompress); + using var tar = new MemoryStream(); + gunzip.CopyTo(tar); + return tar.ToArray(); + } + + /// The previous implementation: compress the whole tar in one write, then normalize the header. + private static byte[] OneShotCompress(byte[] tar) + { + using var output = new MemoryStream(); + using (var gzip = new GZipStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + { + gzip.Write(tar, 0, tar.Length); + } + byte[] bytes = output.ToArray(); + bytes[4] = bytes[5] = bytes[6] = bytes[7] = 0; + bytes[9] = 0xFF; + return bytes; + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs new file mode 100644 index 000000000..eea5a7bca --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs @@ -0,0 +1,418 @@ +using System.Formats.Tar; +using System.IO.Compression; +using AWS.OCI.Builder; +using AWS.OCI.Builder.Images; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class LayerTests : IDisposable +{ + private readonly string _tempRoot; + + public LayerTests() + { + // Each test gets an isolated temp directory for its source files. + _tempRoot = Path.Combine(Path.GetTempPath(), "aws-oci-builder-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempRoot); + } + + public void Dispose() + { + try { Directory.Delete(_tempRoot, recursive: true); } catch { /* best effort cleanup */ } + } + + /// A built layer plus the compressed bytes streamed out. + private sealed record BuiltLayer(Layer Layer, byte[] GzippedTarBytes) + { + public string Digest => Layer.Digest; + public string DiffId => Layer.DiffId; + public long Size => Layer.Size; + public string MediaType => Layer.MediaType; + public Descriptor ToDescriptor() => Layer.ToDescriptor(); + } + + /// Builds a layer into an in-memory destination (fine for these small fixtures). + private static async Task BuildAsync(FileManifest manifest, string mediaType = MediaTypes.OciLayerGzipV1) + { + var destination = new MemoryStream(); + Layer layer = await Layer.CreateAsync(manifest, destination, mediaType); + return new BuiltLayer(layer, destination.ToArray()); + } + + private string WriteFile(string relativePath, string content) + { + string full = Path.Combine(_tempRoot, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(full)!); + File.WriteAllText(full, content); + return full; + } + + [Fact] + public async Task Builds_layer_with_both_digests_populated() + { + string file = WriteFile("app.dll", "hello"); + var manifest = new FileManifest { { file, "/app/app.dll" } }; + + BuiltLayer layer = await BuildAsync(manifest); + + Assert.StartsWith("sha256:", layer.Digest); + Assert.StartsWith("sha256:", layer.DiffId); + Assert.Equal(64 + "sha256:".Length, layer.Digest.Length); + Assert.True(layer.Size > 0); + Assert.Equal(layer.GzippedTarBytes.LongLength, layer.Size); + Assert.Equal(MediaTypes.OciLayerGzipV1, layer.MediaType); + } + + [Fact] + public async Task Digest_and_diffId_differ_because_one_is_compressed() + { + string file = WriteFile("app.dll", new string('x', 4096)); + var manifest = new FileManifest { { file, "/app/app.dll" } }; + + BuiltLayer layer = await BuildAsync(manifest); + + // Digest is over the gzipped bytes; DiffId is over the raw tar. They must not be equal. + Assert.NotEqual(layer.Digest, layer.DiffId); + } + + [Fact] + public async Task Descriptor_reflects_layer_fields() + { + string file = WriteFile("app.dll", "hello"); + var manifest = new FileManifest { { file, "/app/app.dll" } }; + + BuiltLayer layer = await BuildAsync(manifest, MediaTypes.DockerLayerGzip); + Descriptor descriptor = layer.ToDescriptor(); + + Assert.Equal(MediaTypes.DockerLayerGzip, descriptor.MediaType); + Assert.Equal(layer.Size, descriptor.Size); + Assert.Equal(layer.Digest, descriptor.Digest); + } + + [Fact] + public async Task Is_reproducible_identical_inputs_yield_identical_digests() + { + string file = WriteFile("app.dll", "deterministic"); + var manifest1 = new FileManifest { { file, "/app/app.dll" } }; + var manifest2 = new FileManifest { { file, "/app/app.dll" } }; + + BuiltLayer a = await BuildAsync(manifest1); + BuiltLayer b = await BuildAsync(manifest2); + + Assert.Equal(a.Digest, b.Digest); + Assert.Equal(a.DiffId, b.DiffId); + } + + [Fact] + public async Task Entry_order_does_not_affect_digest() + { + string f1 = WriteFile("a.txt", "one"); + string f2 = WriteFile("b.txt", "two"); + + var ordered = new FileManifest { { f1, "/app/a.txt" }, { f2, "/app/b.txt" } }; + var reversed = new FileManifest { { f2, "/app/b.txt" }, { f1, "/app/a.txt" } }; + + BuiltLayer a = await BuildAsync(ordered); + BuiltLayer b = await BuildAsync(reversed); + + Assert.Equal(a.Digest, b.Digest); + } + + [Fact] + public async Task Tar_contains_file_but_not_synthesized_ancestor_directories() + { + string file = WriteFile("app.dll", "hello"); + var manifest = new FileManifest { { file, "/app/bin/app.dll" } }; + + BuiltLayer layer = await BuildAsync(manifest); + List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); + + // The file is present (no leading slash). Its ancestor directories are NOT synthesized as tar entries: + // emitting a 0755 "app/" / "app/bin/" entry would overlay and reset the base image's permissions on those + // directories. Tar extraction creates missing parents implicitly, so the file still lands correctly. + Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/bin/app.dll" }); + Assert.DoesNotContain(entries, e => e is { Type: TarEntryType.Directory, Name: "app/" }); + Assert.DoesNotContain(entries, e => e is { Type: TarEntryType.Directory, Name: "app/bin/" }); + } + + [Fact] + public async Task Directory_mapping_recurses_preserving_subpaths() + { + WriteFile("content/root.txt", "r"); + WriteFile("content/sub/leaf.txt", "l"); + string dir = Path.Combine(_tempRoot, "content"); + + var manifest = new FileManifest { { dir, "/app/data" } }; + + BuiltLayer layer = await BuildAsync(manifest); + List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); + + Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/data/root.txt" }); + Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/data/sub/leaf.txt" }); + Assert.Contains(entries, e => e is { Type: TarEntryType.Directory, Name: "app/data/sub/" }); + } + + [Fact] + public async Task Directory_mapping_preserves_empty_descendant_directories() + { + // A recursive directory mapping must preserve empty subdirectories, not just directories that hold files. + WriteFile("content/root.txt", "r"); + Directory.CreateDirectory(Path.Combine(_tempRoot, "content", "cache")); // empty + string dir = Path.Combine(_tempRoot, "content"); + + var manifest = new FileManifest { { dir, "/app/data" } }; + + BuiltLayer layer = await BuildAsync(manifest); + List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); + + Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/data/root.txt" }); + Assert.Contains(entries, e => e is { Type: TarEntryType.Directory, Name: "app/data/cache/" }); + } + + [Fact] + public async Task Symlinked_source_file_is_rejected() + { + if (OperatingSystem.IsWindows()) + { + return; // Creating symlinks on Windows requires elevation; the guard is exercised on Unix CI. + } + + string real = WriteFile("secret.txt", "host secret"); + string link = Path.Combine(_tempRoot, "link.txt"); + File.CreateSymbolicLink(link, real); + + var manifest = new FileManifest { { link, "/app/link.txt" } }; + + // A symlinked source must be rejected rather than silently followed (which could pull in files from + // outside the mapped set, e.g. host secrets). + await Assert.ThrowsAsync(() => BuildAsync(manifest)); + } + + [Fact] + public async Task Symlink_inside_mapped_directory_is_rejected() + { + if (OperatingSystem.IsWindows()) + { + return; + } + + WriteFile("tree/ok.txt", "ok"); + string outside = WriteFile("outside.txt", "host secret"); + // A link INSIDE the mapped directory pointing outside it must not copy the external file into the layer. + File.CreateSymbolicLink(Path.Combine(_tempRoot, "tree", "escape.txt"), outside); + string dir = Path.Combine(_tempRoot, "tree"); + + var manifest = new FileManifest { { dir, "/app" } }; + + await Assert.ThrowsAsync(() => BuildAsync(manifest)); + } + + [Fact] + public async Task Empty_manifest_throws() + { + var manifest = new FileManifest(); + await Assert.ThrowsAsync(() => BuildAsync(manifest)); + } + + [Fact] + public async Task Missing_source_throws_FileNotFound() + { + var manifest = new FileManifest { { Path.Combine(_tempRoot, "does-not-exist"), "/app/x" } }; + await Assert.ThrowsAsync(() => BuildAsync(manifest)); + } + + [Theory] + [InlineData("relative/path")] // not absolute + [InlineData("/app/../etc")] // escaping segment + [InlineData("/")] // names nothing + public async Task Invalid_container_path_throws(string containerPath) + { + string file = WriteFile("app.dll", "hello"); + var manifest = new FileManifest { { file, containerPath } }; + await Assert.ThrowsAsync(() => BuildAsync(manifest)); + } + + [Fact] + public async Task Duplicate_container_path_throws() + { + string f1 = WriteFile("a.txt", "one"); + string f2 = WriteFile("b.txt", "two"); + var manifest = new FileManifest { { f1, "/app/same.txt" }, { f2, "/app/same.txt" } }; + await Assert.ThrowsAsync(() => BuildAsync(manifest)); + } + + /// Decompresses the gzipped tar and returns its real entries (type + name) in order. + /// + /// PAX archives carry extended-attribute header blocks. Depending on the runtime, + /// surfaces these differently: .NET 9+ folds them into the following entry, while .NET 8's reader can emit a + /// separate pseudo-entry (named PaxHeaders.NNN/...) AND then read the real entry with a stray leading + /// NUL in its name and a non-standard entry-type byte. Those are quirks of the .NET 8 reader, not of the + /// archive bytes (which .NET 9+ and real registries parse correctly). This helper filters the metadata + /// pseudo-entries and normalizes the NUL-prefixed names so the assertions test the layer's real structure. + /// + private static List<(TarEntryType Type, string Name)> ReadTarEntries(byte[] gzippedTar) + => ReadTarEntriesWithMode(gzippedTar).Select(e => (e.Type, e.Name)).ToList(); + + /// Like but also returns each entry's Unix . + private static List<(TarEntryType Type, string Name, UnixFileMode Mode)> ReadTarEntriesWithMode(byte[] gzippedTar) + { + var result = new List<(TarEntryType, string, UnixFileMode)>(); + using var input = new MemoryStream(gzippedTar); + using var gzip = new GZipStream(input, CompressionMode.Decompress); + using var reader = new TarReader(gzip); + while (reader.GetNextEntry() is { } entry) + { + string name = entry.Name.TrimStart('\0'); + + bool isMetadata = + entry.EntryType is TarEntryType.ExtendedAttributes or TarEntryType.GlobalExtendedAttributes + || name.Contains("PaxHeaders", StringComparison.Ordinal); + if (isMetadata) + { + continue; + } + + // Normalize the .NET 8 reader artifact: a name not ending in '/' is a regular file even when the + // reader reports a stray type byte for it. + TarEntryType type = entry.EntryType; + if (type is not TarEntryType.Directory && !name.EndsWith('/')) + { + type = TarEntryType.RegularFile; + } + + result.Add((type, name, entry.Mode)); + } + return result; + } + + // --- File permission coverage (matters especially when building on Windows) --------------- + + [Fact] + public async Task Default_file_mode_is_0644_and_directory_mode_is_0755() + { + string file = WriteFile("app.dll", "hello"); + // Map an explicit directory too, so there is a real (non-synthesized) directory entry to check the mode on. + // Synthesized ancestors are intentionally not emitted, so the directory assertion uses a mapped directory. + string dir = Directory.CreateDirectory(Path.Combine(_tempRoot, "assets")).FullName; + var manifest = new FileManifest { { file, "/app/app.dll" }, { dir, "/app/assets" } }; + + BuiltLayer layer = await BuildAsync(manifest); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + // On Windows the source has no mode bits, so the file defaults to 0644. On Unix the source file written by + // the test is created 0644 by default too (no execute bit), so this holds on every platform. + (TarEntryType Type, string Name, UnixFileMode Mode) fileEntry = + Assert.Single(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/app.dll" }); + Assert.Equal(Mode0644, fileEntry.Mode); + + (TarEntryType Type, string Name, UnixFileMode Mode) dirEntry = + Assert.Single(entries, e => e is { Type: TarEntryType.Directory, Name: "app/assets/" }); + Assert.Equal(Mode0755, dirEntry.Mode); + } + + [Fact] + public async Task Explicit_mode_overrides_the_default() + { + string file = WriteFile("run.sh", "#!/bin/sh\necho hi\n"); + var manifest = new FileManifest { { file, "/app/run.sh", Mode0750 } }; + + BuiltLayer layer = await BuildAsync(manifest); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + var fileEntry = Assert.Single(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/run.sh" }); + Assert.Equal(Mode0750, fileEntry.Mode); + } + + [Fact] + public async Task AddExecutable_marks_the_file_0755() + { + string file = WriteFile("entrypoint.sh", "#!/bin/sh\n"); + var manifest = new FileManifest(); + manifest.AddExecutable(file, "/app/entrypoint.sh"); + + BuiltLayer layer = await BuildAsync(manifest); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + var fileEntry = Assert.Single(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/entrypoint.sh" }); + Assert.Equal(Mode0755, fileEntry.Mode); + } + + [Fact] + public async Task Explicit_mode_applies_to_every_file_in_a_mapped_directory() + { + WriteFile("scripts/a.sh", "#!/bin/sh\n"); + WriteFile("scripts/nested/b.sh", "#!/bin/sh\n"); + string dir = Path.Combine(_tempRoot, "scripts"); + + var manifest = new FileManifest { { dir, "/app/scripts", FileManifestEntry.ExecutableMode } }; + + BuiltLayer layer = await BuildAsync(manifest); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + var a = Assert.Single(entries, e => e.Name == "app/scripts/a.sh"); + var b = Assert.Single(entries, e => e.Name == "app/scripts/nested/b.sh"); + Assert.Equal(Mode0755, a.Mode); + Assert.Equal(Mode0755, b.Mode); + } + + [Fact] + public async Task Executable_bit_from_source_is_preserved_on_unix() + { + // On Windows there is no source executable bit to read, so this behavior only applies to Unix hosts. + if (OperatingSystem.IsWindows()) + { + return; + } + + string file = WriteFile("tool", "#!/bin/sh\necho tool\n"); + File.SetUnixFileMode(file, Mode0755); + + var manifest = new FileManifest { { file, "/app/tool" } }; // no explicit mode -> derive from source + BuiltLayer layer = await BuildAsync(manifest); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + var fileEntry = Assert.Single(entries, e => e.Name == "app/tool"); + // The owner-execute bit from the source must survive into the layer. + Assert.True(fileEntry.Mode.HasFlag(UnixFileMode.UserExecute)); + } + + [Fact] + public async Task Explicit_mode_is_deterministic_in_the_digest() + { + string file = WriteFile("run.sh", "#!/bin/sh\n"); + var m1 = new FileManifest(); + m1.AddExecutable(file, "/app/run.sh"); + var m2 = new FileManifest(); + m2.AddExecutable(file, "/app/run.sh"); + + BuiltLayer a = await BuildAsync(m1); + BuiltLayer b = await BuildAsync(m2); + + Assert.Equal(a.Digest, b.Digest); + } + + [Fact] + public async Task Changing_the_mode_changes_the_digest() + { + string file = WriteFile("run.sh", "#!/bin/sh\n"); + var plain = new FileManifest { { file, "/app/run.sh" } }; + var executable = new FileManifest(); + executable.AddExecutable(file, "/app/run.sh"); + + BuiltLayer a = await BuildAsync(plain); + BuiltLayer b = await BuildAsync(executable); + + // Same content, different permissions -> different layer bytes -> different digest. + Assert.NotEqual(a.Digest, b.Digest); + } + + private const UnixFileMode Mode0644 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead; + private const UnixFileMode Mode0755 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute; + private const UnixFileMode Mode0750 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute; +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs new file mode 100644 index 000000000..17ebbfe77 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs @@ -0,0 +1,756 @@ +using System.Net; +using System.Text; +using AWS.OCI.Builder; +using AWS.OCI.Builder.Images; +using AWS.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class OciImageBuilderTests +{ + /// + /// A minimal in-memory OCI Distribution registry. It tracks blob existence per repository, accepts the + /// POST/PATCH/PUT upload flow, serves base-layer GETs for the source host, and records the PUT manifest. + /// One instance serves all hosts; it keys state by (host, repository). + /// + private sealed class FakeRegistry : HttpMessageHandler + { + // (host, repository) -> set of blob digests present. + private readonly Dictionary<(string Host, string Repo), HashSet> _blobs = new(); + // Pre-seeded source blobs available for download: (host, repo, digest) -> bytes. + private readonly Dictionary<(string Host, string Repo, string Digest), byte[]> _sourceBlobs = new(); + private int _uploadCounter; + + public List<(string Host, string Repo, string Digest)> UploadedBlobs { get; } = new(); + + // Content-Length advertised on each PATCH body (null when the request did not set one). A non-null value + // means HttpClient was able to stream the body instead of buffering it to measure the length. + public List PatchContentLengths { get; } = new(); + + public (string Host, string Repo, string Reference, string Body, string ContentType)? PutManifest { get; private set; } + + public void SeedSourceBlob(string host, string repo, string digest, byte[] bytes) + => _sourceBlobs[(host, repo, digest)] = bytes; + + private HashSet BlobsFor(string host, string repo) + { + if (!_blobs.TryGetValue((host, repo), out HashSet? set)) + { + set = new HashSet(StringComparer.Ordinal); + _blobs[(host, repo)] = set; + } + return set; + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Uri uri = request.RequestUri!; + string host = uri.Host; + string path = uri.AbsolutePath; // /v2//(blobs|manifests|blobs/uploads)/... + HttpMethod method = request.Method; + + // ---- Blob existence: HEAD /v2//blobs/ ---- + if (method == HttpMethod.Head && TryMatch(path, "/blobs/", out string repoHeadBlob, out string headDigest)) + { + bool exists = BlobsFor(host, repoHeadBlob).Contains(headDigest); + return new HttpResponseMessage(exists ? HttpStatusCode.OK : HttpStatusCode.NotFound); + } + + // ---- Blob download: GET /v2//blobs/ ---- + if (method == HttpMethod.Get && TryMatch(path, "/blobs/", out string repoGetBlob, out string getDigest)) + { + if (_sourceBlobs.TryGetValue((host, repoGetBlob, getDigest), out byte[]? bytes)) + { + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(bytes) }; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + // ---- Start upload: POST /v2//blobs/uploads/ ---- + if (method == HttpMethod.Post && path.EndsWith("/blobs/uploads/", StringComparison.Ordinal)) + { + string repo = ExtractRepo(path, "/blobs/uploads/"); + string uploadId = $"upload{Interlocked.Increment(ref _uploadCounter)}"; + var accepted = new HttpResponseMessage(HttpStatusCode.Accepted); + accepted.Headers.Location = new Uri($"https://{host}/v2/{repo}/blobs/uploads/{uploadId}"); + return accepted; + } + + // ---- Chunk: PATCH /v2//blobs/uploads/ ---- + if (method == HttpMethod.Patch && path.Contains("/blobs/uploads/", StringComparison.Ordinal)) + { + // Record the advertised body length before draining (it is cleared once the content is consumed). + PatchContentLengths.Add(request.Content!.Headers.ContentLength); + // Drain the chunk; return the same location for the next chunk/commit. + _ = await request.Content!.ReadAsByteArrayAsync(cancellationToken); + var accepted = new HttpResponseMessage(HttpStatusCode.Accepted); + accepted.Headers.Location = uri; + return accepted; + } + + // ---- Commit: PUT /v2//blobs/uploads/?digest= ---- + if (method == HttpMethod.Put && path.Contains("/blobs/uploads/", StringComparison.Ordinal)) + { + string repo = ExtractRepo(path, "/blobs/uploads/"); + string digest = System.Web.HttpUtility.ParseQueryString(uri.Query).Get("digest")!; + BlobsFor(host, repo).Add(digest); + UploadedBlobs.Add((host, repo, digest)); + return new HttpResponseMessage(HttpStatusCode.Created); + } + + // ---- Manifest PUT: PUT /v2//manifests/ ---- + if (method == HttpMethod.Put && TryMatch(path, "/manifests/", out string repoManifest, out string reference)) + { + string body = await request.Content!.ReadAsStringAsync(cancellationToken); + string contentType = request.Content.Headers.ContentType?.MediaType ?? string.Empty; + PutManifest = (host, repoManifest, reference, body, contentType); + return new HttpResponseMessage(HttpStatusCode.Created); + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + // Matches /v2// and returns repo + tail (tail is the digest or reference). + private static bool TryMatch(string path, string marker, out string repo, out string tail) + { + repo = string.Empty; + tail = string.Empty; + const string prefix = "/v2/"; + int markerIndex = path.IndexOf(marker, StringComparison.Ordinal); + if (!path.StartsWith(prefix, StringComparison.Ordinal) || markerIndex < 0) + { + return false; + } + // Exclude the uploads path from the plain blobs matcher. + if (marker == "/blobs/" && path.Contains("/blobs/uploads/", StringComparison.Ordinal)) + { + return false; + } + repo = path[prefix.Length..markerIndex]; + tail = path[(markerIndex + marker.Length)..]; + return repo.Length > 0 && tail.Length > 0; + } + + private static string ExtractRepo(string path, string marker) + { + const string prefix = "/v2/"; + int markerIndex = path.IndexOf(marker, StringComparison.Ordinal); + return path[prefix.Length..markerIndex]; + } + } + + private const string SourceHost = "mcr.microsoft.com"; + private const string EcrHost = "123456789012.dkr.ecr.us-west-2.amazonaws.com"; + + // A well-formed ECR authorization token: base64("AWS:password"). PushToEcrAsync decodes this into the + // Basic username/password; the fake registry does not check it, so any valid-form token works here. + private static readonly string FakeEcrToken = + Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("AWS:fake-password")); + + /// Builds a base image (config + manifest with one base layer) and seeds the fake source registry. + private static (ImageBuilder Builder, IReadOnlyList BaseLayers) BuildBaseImage(FakeRegistry registry, string sourceRepo) + { + // A base layer blob that CopyBaseLayersAsync must copy to ECR. + byte[] baseLayerBytes = Encoding.UTF8.GetBytes("base-layer-content"); + string baseLayerDigest = RegistryClient.ComputeDigest(new MemoryStream(baseLayerBytes)); + registry.SeedSourceBlob(SourceHost, sourceRepo, baseLayerDigest, baseLayerBytes); + + const string configJson = """ + {"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:aaaa"]},"config":{}} + """; + string manifestJson = $$""" + {"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","size":10,"digest":"sha256:cfg"},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","size":{{baseLayerBytes.Length}},"digest":"{{baseLayerDigest}}"}]} + """; + + ImageBuilder builder = ImageBuilder.FromBaseImage(configJson, manifestJson, MediaTypes.OciManifestV1); + var baseLayers = new[] + { + new Descriptor { MediaType = MediaTypes.OciLayerGzipV1, Size = baseLayerBytes.Length, Digest = baseLayerDigest }, + }; + return (builder, baseLayers); + } + + [Fact] + public async Task PushAsync_copies_base_layers_uploads_new_layer_and_config_and_puts_manifest() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + // Source client reads base-layer blobs; both source and target go through the same fake handler. + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + // Add an application layer from a temp file. + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + var files = new FileManifest { { appFile, "/app/app.txt" } }; + await builder.AddLayerAsync(files, createdBy: "test"); + builder.Config.SetEntrypoint("dotnet", "/app/app.txt"); + + string digest = await builder.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken); + + // The returned digest addresses the manifest. + Assert.StartsWith("sha256:", digest); + + // The manifest was PUT to the ECR target repo under the right tag and media type. + Assert.NotNull(registry.PutManifest); + var put = registry.PutManifest!.Value; + Assert.Equal(EcrHost, put.Host); + Assert.Equal(targetRepo, put.Repo); + Assert.Equal("latest", put.Reference); + Assert.Equal(MediaTypes.OciManifestV1, put.ContentType); + + // All blobs uploaded to ECR: the base layer (copied), the new app layer, and the config. + var ecrUploads = registry.UploadedBlobs.Where(u => u.Host == EcrHost && u.Repo == targetRepo).ToList(); + Manifest pushedManifest = Manifest.Parse(put.Body); + + // Base layer + new layer both present in the manifest and both uploaded. + Assert.Equal(2, pushedManifest.Layers.Count); + foreach (Descriptor layer in pushedManifest.Layers) + { + Assert.Contains(ecrUploads, u => u.Digest == layer.Digest); + } + // Config blob uploaded, and the manifest points at it. + Assert.Contains(ecrUploads, u => u.Digest == pushedManifest.Config.Digest); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_advertises_content_length_on_every_upload_so_bodies_stream() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-cl-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + await builder.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken); + + // Every blob PATCH (base layer, built layer, config) must advertise a Content-Length. A null here + // would mean HttpClient had to buffer the whole body in memory to measure it — the exact memory + // behavior these fixes exist to avoid. + Assert.NotEmpty(registry.PatchContentLengths); + Assert.All(registry.PatchContentLengths, length => Assert.NotNull(length)); + Assert.All(registry.PatchContentLengths, length => Assert.True(length > 0)); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task AddLayerAsync_spills_built_layer_to_disk_and_Dispose_deletes_it() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-spill-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + + // Assert only on the spill files THIS builder owns (from the builder itself), never by scanning the shared + // temp directory: parallel tests spill files with the same name pattern into it concurrently. + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + // Captured before Dispose, which clears the builder's pending-layer list. + IReadOnlyList afterAdd = builder.SpilledLayerPathsForTest; + Assert.Single(afterAdd); + Assert.All(afterAdd, path => Assert.True(File.Exists(path), $"AddLayerAsync should have spilled the built layer to {path}.")); + + // #5: on Unix the spill file must be owner-only (0600), not world-readable (0644). A private layer in + // the shared temp dir must not be readable by other local users. + if (!OperatingSystem.IsWindows()) + { + const UnixFileMode groupOrOther = + UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute; + // A plain loop (not an Assert.All lambda) keeps the call inside the platform guard for CA1416. + foreach (string path in afterAdd) + { + Assert.Equal((UnixFileMode)0, File.GetUnixFileMode(path) & groupOrOther); + } + } + + builder.Dispose(); + + // Every file this test created must be gone after Dispose (regardless of what other tests are doing). + Assert.All(afterAdd, path => Assert.False(File.Exists(path), $"Dispose should have deleted {path}.")); + } + finally + { + builder.Dispose(); + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_skips_base_layer_already_present_in_target() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + await builder.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken); + + // The base layer download count: it must have been copied once (first push). + var put = registry.PutManifest!.Value; + Manifest pushed = Manifest.Parse(put.Body); + string baseLayerDigest = baseLayers[0].Digest; + Assert.Contains(pushed.Layers, l => l.Digest == baseLayerDigest); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_rejects_non_ecr_target() + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync("mcr.microsoft.com/notecr/app:latest", FakeEcrToken)); + } + + [Fact] + public async Task PushAsync_rejects_base_layer_whose_bytes_do_not_match_its_digest() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + // Tamper: make the source registry serve DIFFERENT bytes under the base layer's declared digest, as a + // corrupted cache or a man-in-the-middle-past-TLS registry would. The declared digest is unchanged. + string baseLayerDigest = baseLayers[0].Digest; + registry.SeedSourceBlob(SourceHost, sourceRepo, baseLayerDigest, System.Text.Encoding.UTF8.GetBytes("tampered!")); + + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + // The corrupted base layer must be detected on download and the push aborted. + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken)); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_spills_large_base_layer_to_disk_and_still_pushes_successfully() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + // Force the spill-to-disk path: threshold 0 makes every base layer (size > 0) spill to a temp file, + // get re-verified from disk, and upload from the file rather than memory. + string digest = await builder.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { MaxInMemoryLayerBytes = 0 }); + + Assert.StartsWith("sha256:", digest); + + // The base layer (spilled + verified from disk) and the new layer both reached ECR. + var put = registry.PutManifest!.Value; + Manifest pushed = Manifest.Parse(put.Body); + var ecrUploads = registry.UploadedBlobs.Where(u => u.Host == EcrHost && u.Repo == targetRepo).ToList(); + Assert.Equal(2, pushed.Layers.Count); + foreach (Descriptor layer in pushed.Layers) + { + Assert.Contains(ecrUploads, u => u.Digest == layer.Digest); + } + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_rejects_tampered_base_layer_on_the_spill_to_disk_path() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + // Corrupt the served bytes under the declared digest, exactly as the in-memory tamper test does, but this + // time force the spill path so the disk download + re-read verification is the thing that must catch it. + string baseLayerDigest = baseLayers[0].Digest; + registry.SeedSourceBlob(SourceHost, sourceRepo, baseLayerDigest, System.Text.Encoding.UTF8.GetBytes("tampered!")); + + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { MaxInMemoryLayerBytes = 0 })); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_rejects_negative_in_memory_threshold() + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync( + $"{EcrHost}/myapp:latest", + FakeEcrToken, + new EcrPushOptions { MaxInMemoryLayerBytes = -1 })); + } + + [Theory] + [InlineData("")] // empty + [InlineData(" ")] // whitespace + [InlineData("not-base64-$$$")] // not valid base64 + public async Task PushToEcrAsync_rejects_a_malformed_authorization_token(string badToken) + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync($"{EcrHost}/myapp:latest", badToken)); + } + + [Fact] + public async Task PushToEcrAsync_rejects_a_token_that_decodes_without_a_username_password_separator() + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + // Valid base64, but decodes to "no-separator-here" with no ':' — not a username:password pair. + string tokenWithoutColon = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("no-separator-here")); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync($"{EcrHost}/myapp:latest", tokenWithoutColon)); + } + + [Theory] + [InlineData(":password", "username")] // empty username + [InlineData("AWS:", "password")] // empty password + public async Task PushToEcrAsync_rejects_a_token_with_an_empty_username_or_password(string decodedToken, string emptyPart) + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + string token = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(decodedToken)); + + ArgumentException ex = await Assert.ThrowsAsync( + () => builder.PushToEcrAsync($"{EcrHost}/myapp:latest", token)); + Assert.Contains($"empty {emptyPart}", ex.Message, StringComparison.Ordinal); + } + + // ---- Progress reporting ---------------------------------------------------------------------- + + /// Builds a fresh builder over the shared fake registry with one base layer and one app layer. + private static async Task BuildTwoLayerImageAsync(FakeRegistry registry, string sourceRepo, string tempDir) + { + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello-progress"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + return builder; + } + + [Fact] + public async Task PushToEcrAsync_reports_progress_for_every_layer_with_stable_index_total_and_kind() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + using OciImageBuilder builder = await BuildTwoLayerImageAsync(registry, sourceRepo, tempDir); + + var events = new List(); + await builder.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { Progress = new SynchronousProgress(events.Add) }); + + // Two layers: index 1 = base layer, index 2 = built app layer; TotalLayers is 2 on every event. + Assert.All(events, e => Assert.Equal(2, e.TotalLayers)); + Assert.All(events, e => Assert.InRange(e.LayerIndex, 1, 2)); + + // The config blob and manifest are NOT layers, so they produce no events — only layer indices appear. + Assert.Equal(new[] { 1, 2 }, events.Select(e => e.LayerIndex).Distinct().OrderBy(i => i).ToArray()); + + // Layer 1 is the base layer; layer 2 is the locally-built layer. Kind is stable per index. + Assert.All(events.Where(e => e.LayerIndex == 1), e => Assert.Equal(LayerKind.BaseLayer, e.Kind)); + Assert.All(events.Where(e => e.LayerIndex == 2), e => Assert.Equal(LayerKind.BuiltLayer, e.Kind)); + + // Each layer has exactly one digest across all its events (stable identity). + foreach (int index in new[] { 1, 2 }) + { + Assert.Single(events.Where(e => e.LayerIndex == index).Select(e => e.Digest).Distinct()); + } + + // Every layer terminates on exactly one Completed event — one consistent "done" signal. + Assert.Equal(2, events.Count(e => e.Phase == LayerPushPhase.Completed)); + foreach (int index in new[] { 1, 2 }) + { + Assert.Single(events, e => e.LayerIndex == index && e.Phase == LayerPushPhase.Completed); + } + + // Each uploaded layer starts before it completes. + foreach (int index in new[] { 1, 2 }) + { + var perLayer = events.Where(e => e.LayerIndex == index).ToList(); + int startedAt = perLayer.FindIndex(e => e.Phase == LayerPushPhase.Started); + int completedAt = perLayer.FindIndex(e => e.Phase == LayerPushPhase.Completed); + Assert.True(startedAt >= 0, $"layer {index} should have a Started event"); + Assert.True(startedAt < completedAt, $"layer {index} Started must precede Completed"); + } + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushToEcrAsync_reports_a_skipped_layer_as_a_single_completed_event_with_IsSkipped_true() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + // First push lands the base layer + app layer in the ECR target repo. + using (OciImageBuilder first = await BuildTwoLayerImageAsync(registry, sourceRepo, tempDir)) + { + await first.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken); + } + + // Second push of an identical image to the SAME target: both layers already exist, so both are skipped. + using OciImageBuilder second = await BuildTwoLayerImageAsync(registry, sourceRepo, tempDir); + var events = new List(); + await second.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { Progress = new SynchronousProgress(events.Add) }); + + // Each layer is reported as exactly one event: Completed + IsSkipped, with no preceding Started/Uploading. + foreach (int index in new[] { 1, 2 }) + { + var perLayer = events.Where(e => e.LayerIndex == index).ToList(); + LayerPushProgress only = Assert.Single(perLayer); + Assert.Equal(LayerPushPhase.Completed, only.Phase); + Assert.True(only.IsSkipped); + Assert.Equal(0, only.BytesPushed); + // LayerSizeBytes is still the real size (so a UI can say "skipped — N bytes already present"). + Assert.True(only.LayerSizeBytes > 0); + } + + // No Started or Uploading events at all when everything is skipped. + Assert.DoesNotContain(events, e => e.Phase == LayerPushPhase.Started); + Assert.DoesNotContain(events, e => e.Phase == LayerPushPhase.Uploading); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushToEcrAsync_reports_byte_progress_reaching_layer_size_at_completion() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + using OciImageBuilder builder = await BuildTwoLayerImageAsync(registry, sourceRepo, tempDir); + + var events = new List(); + await builder.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { Progress = new SynchronousProgress(events.Add) }); + + foreach (int index in new[] { 1, 2 }) + { + var perLayer = events.Where(e => e.LayerIndex == index).ToList(); + + LayerPushProgress started = perLayer.First(e => e.Phase == LayerPushPhase.Started); + Assert.Equal(0, started.BytesPushed); + + // Uploading events (if any) never exceed the layer size and are monotonically non-decreasing. + var uploading = perLayer.Where(e => e.Phase == LayerPushPhase.Uploading).ToList(); + long previous = 0; + foreach (LayerPushProgress u in uploading) + { + Assert.True(u.BytesPushed >= previous, "byte progress must not go backward"); + Assert.True(u.BytesPushed <= u.LayerSizeBytes, "byte progress must not exceed the layer size"); + previous = u.BytesPushed; + } + + // At completion, bytes pushed equals the full layer size. + LayerPushProgress completed = perLayer.Single(e => e.Phase == LayerPushPhase.Completed); + Assert.Equal(completed.LayerSizeBytes, completed.BytesPushed); + Assert.False(completed.IsSkipped); + } + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + /// + /// A deterministic that invokes the callback synchronously on the reporting thread, + /// so a test observes every event in emission order (unlike , which marshals and may + /// reorder/defer via the captured synchronization context). + /// + private sealed class SynchronousProgress : IProgress + { + private readonly Action _onReport; + public SynchronousProgress(Action onReport) => _onReport = onReport; + public void Report(LayerPushProgress value) => _onReport(value); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs new file mode 100644 index 000000000..2e0f448ff --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs @@ -0,0 +1,66 @@ +using AWS.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class RealmValidatorTests +{ + [Theory] + [InlineData("https://auth.docker.io/token")] + [InlineData("https://public.ecr.aws/token/")] + [InlineData("https://123456789012.dkr.ecr.us-west-2.amazonaws.com/")] + public void Accepts_public_https_realms(string realm) + { + Uri result = RealmValidator.Validate(realm, "registry.example.com"); + Assert.Equal("https", result.Scheme); + } + + [Theory] + [InlineData("http://auth.docker.io/token")] // not https + [InlineData("ftp://auth.docker.io/token")] // not https + public void Rejects_non_https_scheme(string realm) + { + Assert.Throws(() => RealmValidator.Validate(realm, "r")); + } + + [Theory] + [InlineData("https://127.0.0.1/token")] // loopback v4 + [InlineData("https://[::1]/token")] // loopback v6 + [InlineData("https://10.0.0.5/token")] // private + [InlineData("https://192.168.1.10/token")] // private + [InlineData("https://172.16.4.4/token")] // private + [InlineData("https://169.254.1.1/token")] // link-local + [InlineData("https://localhost/token")] // loopback name + [InlineData("https://foo.localhost/token")] // loopback name + [InlineData("https://127.0.0.1./token")] // trailing-dot loopback + public void Rejects_loopback_private_and_linklocal_hosts(string realm) + { + Assert.Throws(() => RealmValidator.Validate(realm, "r")); + } + + [Fact] + public void Rejects_unparseable_realm() + { + Assert.Throws(() => RealmValidator.Validate("not a url", "r")); + } + + [Theory] + [InlineData("127.0.0.1", true)] // loopback + [InlineData("10.1.2.3", true)] // private + [InlineData("172.16.5.5", true)] // private + [InlineData("192.168.0.9", true)] // private + [InlineData("169.254.1.1", true)] // link-local + [InlineData("::1", true)] // IPv6 loopback + [InlineData("8.8.8.8", false)] // public + public void IsBlockedAddress_flags_private_and_loopback(string ip, bool expected) + => Assert.Equal(expected, RealmValidator.IsBlockedAddress(System.Net.IPAddress.Parse(ip))); + + [Theory] + [InlineData("https://localhost/x", true)] + [InlineData("https://sub.localhost/x", true)] + [InlineData("https://127.0.0.1/x", true)] + [InlineData("https://10.0.0.5/x", true)] + [InlineData("https://registry.example.com/x", false)] // public DNS name: not blocked by the pre-connect check + public void IsBlockedHost_flags_loopback_names_and_blocked_literals(string url, bool expected) + => Assert.Equal(expected, RealmValidator.IsBlockedHost(new Uri(url))); +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs new file mode 100644 index 000000000..eb6260333 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs @@ -0,0 +1,79 @@ +using AWS.OCI.Builder; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class UnixPermissionsTests +{ + private const UnixFileMode Mode0755 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute; + private const UnixFileMode Mode0644 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead; + private const UnixFileMode Mode0600 = + UnixFileMode.UserRead | UnixFileMode.UserWrite; + + [Theory] + [InlineData("755")] + [InlineData("0755")] + [InlineData("0o755")] + [InlineData("0O755")] + [InlineData(" 755 ")] + public void FromOctal_string_parses_755(string input) + => Assert.Equal(Mode0755, UnixPermissions.FromOctal(input)); + + [Theory] + [InlineData("644", 0b110_100_100)] + [InlineData("600", 0b110_000_000)] + [InlineData("777", 0b111_111_111)] + [InlineData("000", 0)] + public void FromOctal_string_maps_bits_correctly(string input, int expectedBits) + => Assert.Equal((UnixFileMode)expectedBits, UnixPermissions.FromOctal(input)); + + [Fact] + public void FromOctal_known_values() + { + Assert.Equal(Mode0644, UnixPermissions.FromOctal("644")); + Assert.Equal(Mode0600, UnixPermissions.FromOctal("600")); + } + + [Theory] + [InlineData(755)] + [InlineData(644)] + [InlineData(600)] + public void FromOctal_int_matches_string(int value) + => Assert.Equal(UnixPermissions.FromOctal(value.ToString()), UnixPermissions.FromOctal(value)); + + [Fact] + public void FromOctal_int_literal_755_is_octal_0755() + => Assert.Equal(Mode0755, UnixPermissions.FromOctal(755)); + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("75")] // too few digits + [InlineData("7555")] // too many (non-zero leading) + [InlineData("788")] // 8 is not an octal digit + [InlineData("7x5")] // non-digit + public void FromOctal_string_rejects_invalid(string input) + => Assert.Throws(() => UnixPermissions.FromOctal(input)); + + [Theory] + [InlineData(-1)] + [InlineData(778)] // digit 8 out of octal range + [InlineData(800)] // > 777 + [InlineData(1000)] // out of range + public void FromOctal_int_rejects_invalid(int value) + => Assert.Throws(() => UnixPermissions.FromOctal(value)); + + [Fact] + public void FromOctal_result_is_usable_as_an_explicit_entry_mode() + { + // Confirms the helper's output plugs straight into the FileManifest mode overload. + var files = new FileManifest(); + files.Add("/some/local/path", "/app/tool", UnixPermissions.FromOctal("750")); + + Assert.Single(files.Entries); + Assert.Equal(UnixPermissions.FromOctal("750"), files.Entries[0].Mode); + } +} diff --git a/buildtools/build.proj b/buildtools/build.proj index 32f156103..337628854 100644 --- a/buildtools/build.proj +++ b/buildtools/build.proj @@ -220,6 +220,7 @@ +