Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions Sources/Containerization/ContainerManager.swift
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,50 @@ public struct ContainerManager: Sendable {
}
}

/// Returns a new container from the provided image and root filesystem mount, with a new, empty
/// writable layer in the container's directory.
///
/// Use this for a root filesystem that is not a file in the container's directory, for example
/// an image's snapshot served over a network block device. The root filesystem is the read-only
/// lower layer of an overlayfs, and the container writes to the writable layer.
/// - Parameters:
/// - id: The container ID.
/// - image: The image.
/// - rootfs: The root filesystem mount. The `destination` field is ignored as mounting is handled internally.
/// - writableLayerSizeInBytes: The size of the writable layer that holds the container's writes.
/// - networking: Whether to create a network interface for this container. Defaults to `true`.
/// When `false`, no network resources are allocated and `releaseNetwork`/`delete` remain safe to call.
/// - vm: The size of the VM the container runs in.
public mutating func create(
_ id: String,
image: Image,
rootfs: Mount,
writableLayerSizeInBytes: UInt64,
networking: Bool = true,
vm: VMResources = .default,
configuration: (inout LinuxContainer.Configuration) throws -> Void
) async throws -> LinuxContainer {
let path = try createContainerRoot(id)
do {
let writableLayer = try createEmptyFilesystem(
at: path.appendingPathComponent("writable.ext4"),
size: writableLayerSizeInBytes
)
return try await create(
id,
image: image,
rootfs: rootfs,
writableLayer: writableLayer,
networking: networking,
vm: vm,
configuration: configuration
)
} catch {
try? FileManager.default.removeItem(at: path)
throw error
}
}

/// Returns a new container from the provided image and root filesystem mount.
/// - Parameters:
/// - id: The container ID.
Expand Down
46 changes: 35 additions & 11 deletions Sources/Containerization/Image/ImageStore/ImageStore+Import.swift
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,28 @@ extension ImageStore {
let progress: ProgressHandler?
let name: String
let maxConcurrentDownloads: Int
let reusesStoredContent: Bool

public init(name: String, contentStore: ContentStore, client: ContentClient, ingestDir: URL, progress: ProgressHandler? = nil, maxConcurrentDownloads: Int = 3) {
self.init(
name: name, contentStore: contentStore, client: client, ingestDir: ingestDir, progress: progress, maxConcurrentDownloads: maxConcurrentDownloads,
reusesStoredContent: true)
}

/// - Parameter reusesStoredContent: Whether to take blobs the content store already has
/// instead of fetching them from the client. With `false`, every blob must come from the
/// client, so the source cannot use content it does not hold.
init(
name: String, contentStore: ContentStore, client: ContentClient, ingestDir: URL, progress: ProgressHandler? = nil, maxConcurrentDownloads: Int = 3,
reusesStoredContent: Bool
) {
self.client = client
self.ingestDir = ingestDir
self.contentStore = contentStore
self.progress = progress
self.name = name
self.maxConcurrentDownloads = maxConcurrentDownloads
self.reusesStoredContent = reusesStoredContent
}

/// Pull the required image layers for the provided descriptor and platform(s) into the given directory using the provided client. Returns a descriptor to the Index manifest.
Expand Down Expand Up @@ -101,10 +115,10 @@ extension ImageStore {
private func getManifestContent<T: Sendable & Codable>(descriptor: Descriptor) async throws -> T {
do {
let encoded = try descriptor.digest.validatedDigestEncoding()
if let content = try await self.contentStore.get(digest: descriptor.digest) {
if let content = try? LocalContent(path: ingestDir.appending(path: encoded)) {
return try content.decode()
}
if let content = try? LocalContent(path: ingestDir.appending(path: encoded)) {
if self.reusesStoredContent, let content = try await self.contentStore.get(digest: descriptor.digest) {
return try content.decode()
}
return try await self.client.fetch(name: name, descriptor: descriptor)
Expand Down Expand Up @@ -184,15 +198,25 @@ extension ImageStore {

private func fetch(_ descriptor: Descriptor) async throws {
let encoded = try descriptor.digest.validatedDigestEncoding()
if let found = try await self.contentStore.get(digest: descriptor.digest) {
try FileManager.default.copyItem(at: found.path, to: ingestDir.appendingPathComponent(encoded))
await progress?([
// Count the size of the blob
.addSize(descriptor.size),
// Count the number of blobs
.addItems(1),
])
return
if self.reusesStoredContent, let found = try await self.contentStore.get(digest: descriptor.digest) {
let destination = ingestDir.appendingPathComponent(encoded)
do {
try FileManager.default.copyItem(at: found.path, to: destination)
await progress?([
// Count the size of the blob
.addSize(descriptor.size),
// Count the number of blobs
.addItems(1),
])
return
} catch {
// A garbage collection of the content store can delete the blob after the lookup.
// Then fetch it from the source like any blob the store does not have.
guard !FileManager.default.fileExists(atPath: found.path.path) else {
throw error
}
try? FileManager.default.removeItem(at: destination)
}
}

if descriptor.size > 1.mib() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,30 @@ extension ImageStore {
try client.createOCILayoutStructure(directory: out, manifests: saved)
}

/// Imports the one image in an OCI Image Layout directory and stores it under `reference`.
///
/// Unlike ``load(from:progress:)``, this ignores the layout's names and takes every blob from the
/// layout, never from the content store, so an untrusted layout cannot replace another image or
/// use another image's layers. An image already stored under `reference` is replaced.
///
/// - Parameters:
/// - directory: A URL to a directory on disk that follows the OCI Image Layout structure.
/// - reference: The name to store the image under.
/// - platform: The platform to import. Defaults to `nil`, which imports every platform in the layout.
/// - progress: An optional handler over which progress update events about the load operation can be received.
/// - Returns: The loaded image.
/// - Throws: A `ContainerizationError` with code `.invalidArgument` if the layout does not hold exactly one image.
public func load(from directory: URL, as reference: String, platform: Platform? = nil, progress: ProgressHandler? = nil) async throws -> Image {
let client = try LocalOCILayoutClient(root: directory)
let manifests = try client.loadIndexFromOCILayout(directory: directory).manifests
guard manifests.count == 1, let root = manifests.first else {
throw ContainerizationError(.invalidArgument, message: "OCI layout at \(directory.path) holds \(manifests.count) images, not 1")
}
return try await self.load(
[(root, reference)], from: client, matcher: createPlatformMatcher(for: platform), reusesStoredContent: false, progress: progress
)[0]
}

/// Imports one or more images and their associated layers from an OCI Image Layout directory.
///
/// - Parameters:
Expand All @@ -74,22 +98,35 @@ extension ImageStore {
public func load(from directory: URL, progress: ProgressHandler? = nil) async throws -> [Image] {
let client = try LocalOCILayoutClient(root: directory)
let index = try client.loadIndexFromOCILayout(directory: directory)
let matcher = createPlatformMatcher(for: nil)
let roots = index.manifests.map { ($0, client.getImageReferencefromDescriptor(descriptor: $0)) }
let images = try await self.load(roots, from: client, matcher: createPlatformMatcher(for: nil), reusesStoredContent: true, progress: progress)
guard images.count > 0 else {
throw ContainerizationError(.internalError, message: "failed to import image")
}
return images
}

/// Imports each root into one ingest session, then stores each image under its reference.
private func load(
_ roots: [(descriptor: Descriptor, reference: String)],
from client: LocalOCILayoutClient,
matcher: @Sendable (Platform) -> Bool,
reusesStoredContent: Bool,
progress: ProgressHandler?
) async throws -> [Image] {
var loaded: [Image.Description] = []
let (id, tempDir) = try await self.contentStore.newIngestSession()
do {
for descriptor in index.manifests {
let reference = client.getImageReferencefromDescriptor(descriptor: descriptor)
let ref = try Reference.parse(reference)
let name = ref.path
let operation = ImportOperation(name: name, contentStore: self.contentStore, client: client, ingestDir: tempDir, progress: progress)
let indexDesc = try await operation.import(root: descriptor, matcher: matcher)
loaded.append(Image.Description(reference: reference, descriptor: indexDesc))
for root in roots {
let name = try Reference.parse(root.reference).path
let operation = ImportOperation(
name: name, contentStore: self.contentStore, client: client, ingestDir: tempDir, progress: progress, reusesStoredContent: reusesStoredContent)
let indexDesc = try await operation.import(root: root.descriptor, matcher: matcher)
loaded.append(Image.Description(reference: root.reference, descriptor: indexDesc))
}

let loadedImages = loaded
let importedImages = try await self.lock.withLock { lock in
return try await self.lock.withLock { lock in
var images: [Image] = []
try await self.contentStore.completeIngestSession(id)
for description in loadedImages {
Expand All @@ -98,10 +135,6 @@ extension ImageStore {
}
return images
}
guard importedImages.count > 0 else {
throw ContainerizationError(.internalError, message: "failed to import image")
}
return importedImages
} catch {
try? await self.contentStore.cancelIngestSession(id)
throw error
Expand Down
53 changes: 53 additions & 0 deletions Tests/ContainerizationTests/ContainerManagerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,59 @@ struct ContainerManagerTests {
#expect(closureWasCalled, "configuration closure must be invoked to validate interfaces")
}

@Test func testCreateFromRootfsAddsAnEmptyWritableLayerInTheContainerDirectory() async throws {
let fm = FileManager.default
let root = fm.uniqueTemporaryDirectory(create: true)
defer { try? fm.removeItem(at: root) }

let kernelPath = root.appendingPathComponent("vmlinux")
fm.createFile(atPath: kernelPath.path, contents: Data(), attributes: nil)
let initfsPath = root.appendingPathComponent("initfs.ext4")
fm.createFile(atPath: initfsPath.path, contents: Data(), attributes: nil)

let kernel = Kernel(path: kernelPath, platform: .linuxArm)
let initfs = Mount.block(format: "ext4", source: initfsPath.path, destination: "/")

var manager = try ContainerManager(kernel: kernel, initfs: initfs, root: root, network: NilGatewayNetwork())

let tempDir = fm.uniqueTemporaryDirectory()
defer { try? fm.removeItem(at: tempDir) }

let tarPath = try #require(Foundation.Bundle.module.url(forResource: "scratch", withExtension: "tar"))
let reader = try ArchiveReader(format: .pax, filter: .none, file: tarPath)
let rejectedPaths = try reader.extractContents(to: tempDir)
#expect(rejectedPaths.isEmpty)

let images = try await manager.imageStore.load(from: tempDir)
let image = try #require(images.first)

// Stands in for a root filesystem outside the container's directory, such as an NBD export.
let rootfsPath = root.appendingPathComponent("shared.ext4")
fm.createFile(atPath: rootfsPath.path, contents: Data(), attributes: nil)
let rootfs = Mount.block(format: "ext4", source: rootfsPath.path, destination: "/")

let container = try await manager.create(
"rootfs-writable",
image: image,
rootfs: rootfs,
writableLayerSizeInBytes: 16.mib(),
networking: false
) { _ in }

let directory = root.appendingPathComponent("containers").appendingPathComponent("rootfs-writable")
let writableLayer = try #require(container.writableLayer)
#expect(container.rootfs.source == rootfsPath.path)
#expect(writableLayer.source == directory.appendingPathComponent("writable.ext4").absolutePath())
#expect(try !EXT4.EXT4Reader(blockDevice: FilePath(writableLayer.source)).exists(FilePath("/file")))

// A failed create removes the container's directory, so the ID can be used again. With
// networking, NilGatewayNetwork makes the create fail after the directory exists.
await #expect(throws: ContainerizationError.self) {
_ = try await manager.create("rootfs-failed", image: image, rootfs: rootfs, writableLayerSizeInBytes: 16.mib()) { _ in }
}
#expect(!fm.fileExists(atPath: root.appendingPathComponent("containers").appendingPathComponent("rootfs-failed").path))
}

@Test func testCreateFromSnapshotClonesTheSnapshotIntoTheContainerDirectory() async throws {
let fm = FileManager.default
let root = fm.uniqueTemporaryDirectory(create: true)
Expand Down
58 changes: 58 additions & 0 deletions Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,64 @@ public class ImageStoreTests: ContainsAuth {
#expect(message.contains(Self.scratchAmd64Manifest), "\(message)")
}

@Test func testLoadAsReferenceIgnoresTheLayoutsName() async throws {
let tempDir = try extractScratch(removing: [])
defer {
try? FileManager.default.removeItem(at: tempDir)
}

let arm64 = ContainerizationOCI.Platform(arch: "arm64", os: "linux")
let image = try await self.store.load(from: tempDir, as: "registry.local/chosen:1", platform: arm64)
#expect(image.reference == "registry.local/chosen:1")
_ = try await image.manifest(for: arm64)

// The name in the layout is never stored, so the layout cannot replace an image stored under it.
#expect(try await self.store.getOrNil(reference: "registry.local/integration-tests/scratch:latest") == nil)
// Only the requested platform is imported.
let amd64 = ContainerizationOCI.Platform(arch: "amd64", os: "linux")
await #expect(throws: (any Error).self) {
_ = try await image.manifest(for: amd64)
}
}

@Test func testLoadAsReferenceRejectsALayoutWithMoreThanOneImage() async throws {
let tempDir = try extractScratch(removing: [])
defer {
try? FileManager.default.removeItem(at: tempDir)
}
let indexPath = tempDir.appending(path: "index.json")
var index = try JSONDecoder().decode(ContainerizationOCI.Index.self, from: Data(contentsOf: indexPath))
index = ContainerizationOCI.Index(manifests: index.manifests + index.manifests)
try JSONEncoder().encode(index).write(to: indexPath)

let error = await #expect(throws: ContainerizationError.self) {
_ = try await self.store.load(from: tempDir, as: "registry.local/chosen:1")
}
#expect(error?.code == .invalidArgument)
#expect(try await self.store.getOrNil(reference: "registry.local/chosen:1") == nil)
}

@Test func testLoadAsReferenceTakesNoContentFromTheStore() async throws {
// The store holds the image's layer, from an image loaded before.
let full = try extractScratch(removing: [])
defer {
try? FileManager.default.removeItem(at: full)
}
_ = try await self.store.load(from: full)

// A layout that names the layer without holding it cannot have the store fill it in.
let layer = "530a471644602a73c05568aa468506f7a630f6c0a0e43e9052f1f4469d8241cf"
let partial = try extractScratch(removing: [layer])
defer {
try? FileManager.default.removeItem(at: partial)
}
let arm64 = ContainerizationOCI.Platform(arch: "arm64", os: "linux")
await #expect(throws: (any Error).self) {
_ = try await self.store.load(from: partial, as: "registry.local/chosen:1", platform: arm64)
}
#expect(try await self.store.getOrNil(reference: "registry.local/chosen:1") == nil)
}

@Test func testGetOrNilMissingReference() async throws {
#expect(try await self.store.getOrNil(reference: "registry.local/missing:latest") == nil)
}
Expand Down
Loading
Loading