Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions ManagedCode.Storage.Core/Primitives/VerifiedObjectUpload.cs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ public static async Task<VerifiedObjectUploadResult> WriteIfAbsentOrSameAsync(
if (info.Length != expectedLength ||
(options.ContentType is not null &&
!string.Equals(info.ContentType, options.ContentType, StringComparison.OrdinalIgnoreCase)) ||
(options.ContentEncoding is not null &&
!string.Equals(info.ContentEncoding, options.ContentEncoding, StringComparison.OrdinalIgnoreCase)) ||
(options.Metadata is not null && !MetadataMatches(options.Metadata, info.Metadata)) ||
!await MatchesExistingAsync(multipart, path, info.ETag, expectedLength,
inputHash, cancellationToken).ConfigureAwait(false))
{
Expand Down Expand Up @@ -138,6 +141,42 @@ private static string PartId(byte[] transferId, int index)
return Convert.ToBase64String(value);
}

private static bool MetadataMatches(IReadOnlyDictionary<string, string> expected,
IReadOnlyDictionary<string, string> actual)
{
if (expected.Count != actual.Count)
{
return false;
}

foreach (var item in expected)
{
var found = false;
foreach (var stored in actual)
{
if (!string.Equals(item.Key, stored.Key, StringComparison.OrdinalIgnoreCase))
{
continue;
}

if (!string.Equals(item.Value, stored.Value, StringComparison.Ordinal))
{
return false;
}

found = true;
break;
}

if (!found)
{
return false;
}
}

return true;
}

private static async Task<bool> MatchesExistingAsync(IObjectStorage storage, string path,
string etag, long expectedLength, byte[] inputHash, CancellationToken cancellationToken)
{
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ flowchart LR

Keyed provider registrations let you resolve multiple named instances from dependency injection while reusing the same abstraction across Azure, AWS, Google Cloud Storage, Google Drive, OneDrive, Dropbox, CloudKit, SFTP, and local file system storage.

Immutable uploads can call `WriteIfAbsentOrSameAsync` on `IObjectStorage` with a declared length. The operation streams and verifies the content, and an identical retry returns the existing object revision. A different payload keeps the storage conflict.
Immutable uploads can call `WriteIfAbsentOrSameAsync` on `IObjectStorage` with a declared length. The operation streams and verifies the content, and an identical retry with matching content type, encoding, and metadata returns the existing object revision. A different payload or placement metadata keeps the storage conflict.

### ASP.NET Streaming Controllers

Expand Down
2 changes: 1 addition & 1 deletion Storages/ManagedCode.Storage.CloudKit/CloudKitStorage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ protected override async Task<Result<LocalFile>> DownloadInternalAsync(LocalFile
var remoteStream = await StorageClient.DownloadAsync(recordName, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();

var fileStream = localFile.FileStream;
await using (remoteStream)
await using (var fileStream = localFile.FileStream)
{
await remoteStream.CopyToAsync(fileStream, cancellationToken);
fileStream.Position = 0;
Expand Down
2 changes: 1 addition & 1 deletion Storages/ManagedCode.Storage.Dropbox/DropboxStorage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,8 @@ protected override async Task<Result<LocalFile>> DownloadInternalAsync(LocalFile
var path = BuildFullPath(options.FullPath);
var remoteStream = await StorageClient.DownloadAsync(StorageOptions.RootPath, path, cancellationToken);

var fileStream = localFile.FileStream;
await using (remoteStream)
await using (var fileStream = localFile.FileStream)
{
await remoteStream.CopyToAsync(fileStream, cancellationToken);
fileStream.Position = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@ protected override async Task<Result<LocalFile>> DownloadInternalAsync(LocalFile
var path = BuildFullPath(options.FullPath);
var remoteStream = await StorageClient.DownloadAsync(StorageOptions.RootFolderId, path, StorageOptions.SupportsAllDrives, cancellationToken);

var fileStream = localFile.FileStream;
await using (remoteStream)
await using (var fileStream = localFile.FileStream)
{
await remoteStream.CopyToAsync(fileStream, cancellationToken);
fileStream.Position = 0;
Expand Down
2 changes: 1 addition & 1 deletion Storages/ManagedCode.Storage.OneDrive/OneDriveStorage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,8 @@ protected override async Task<Result<LocalFile>> DownloadInternalAsync(LocalFile
var remoteStream = await StorageClient.DownloadAsync(StorageOptions.DriveId, path, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();

var fileStream = localFile.FileStream;
await using (remoteStream)
await using (var fileStream = localFile.FileStream)
{
await remoteStream.CopyToAsync(fileStream, cancellationToken);
fileStream.Position = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,23 +78,75 @@ public async Task VerifiedUpload_RetryWithSameContentReusesRevision_AndDifferent
using var storage = CreateStorage();
var objects = storage.RequireObjectStorage();
await objects.CreatePrivateContainerAsync();
var metadata = new Dictionary<string, string>
{
["owner"] = "company-a",
["generation"] = "1"
};
using var firstContent = Content("original");
var first = await objects.WriteIfAbsentOrSameAsync("file.txt", firstContent, 8,
new StorageWriteOptions { ContentType = "text/plain" });
new StorageWriteOptions { ContentType = "text/plain", Metadata = metadata });
first.ReusedExisting.ShouldBeFalse();
using var retryContent = Content("original");
var retry = await objects.WriteIfAbsentOrSameAsync("file.txt", retryContent, 8,
new StorageWriteOptions { ContentType = "text/plain" });
new StorageWriteOptions { ContentType = "text/plain", Metadata = metadata });
retry.ReusedExisting.ShouldBeTrue();
retry.Info.ETag.ShouldBe(first.Info.ETag);
retry.Sha256.ShouldBe(first.Sha256);
using var differentlyCasedMetadata = Content("original");
var sameMetadata = await objects.WriteIfAbsentOrSameAsync("file.txt", differentlyCasedMetadata, 8,
new StorageWriteOptions
{
ContentType = "text/plain",
Metadata = new Dictionary<string, string>
{
["Owner"] = "company-a",
["Generation"] = "1"
}
});
sameMetadata.ReusedExisting.ShouldBeTrue();
sameMetadata.Info.ETag.ShouldBe(first.Info.ETag);
using var wrongType = Content("original");
(await Should.ThrowAsync<StorageOperationException>(() =>
objects.WriteIfAbsentOrSameAsync("file.txt", wrongType, 8,
new StorageWriteOptions { ContentType = "application/json" }))).IsConflict.ShouldBeTrue();
using var differentContent = Content("different");
using var wrongOwner = Content("original");
(await Should.ThrowAsync<StorageOperationException>(() =>
objects.WriteIfAbsentOrSameAsync("file.txt", wrongOwner, 8,
new StorageWriteOptions
{
ContentType = "text/plain",
Metadata = new Dictionary<string, string>
{
["owner"] = "company-b",
["generation"] = "1"
}
}))).IsConflict.ShouldBeTrue();
using var wrongMetadataCount = Content("original");
(await Should.ThrowAsync<StorageOperationException>(() =>
objects.WriteIfAbsentOrSameAsync("file.txt", wrongMetadataCount, 8,
new StorageWriteOptions
{
Metadata = new Dictionary<string, string> { ["owner"] = "company-a" }
}))).IsConflict.ShouldBeTrue();
using var wrongMetadataKey = Content("original");
(await Should.ThrowAsync<StorageOperationException>(() =>
objects.WriteIfAbsentOrSameAsync("file.txt", wrongMetadataKey, 8,
new StorageWriteOptions
{
Metadata = new Dictionary<string, string>
{
["tenant"] = "company-a",
["generation"] = "1"
}
}))).IsConflict.ShouldBeTrue();
using var wrongEncoding = Content("original");
(await Should.ThrowAsync<StorageOperationException>(() =>
objects.WriteIfAbsentOrSameAsync("file.txt", wrongEncoding, 8,
new StorageWriteOptions { ContentEncoding = "gzip" }))).IsConflict.ShouldBeTrue();
using var differentContent = Content("altered!");
(await Should.ThrowAsync<StorageOperationException>(() =>
objects.WriteIfAbsentOrSameAsync("file.txt", differentContent, 9))).IsConflict.ShouldBeTrue();
objects.WriteIfAbsentOrSameAsync("file.txt", differentContent, 8))).IsConflict.ShouldBeTrue();
await using var stored = await objects.OpenObjectReadAsync("file.txt");
using var reader = new StreamReader(stored);
(await reader.ReadToEndAsync()).ShouldBe("original");
Expand Down
2 changes: 1 addition & 1 deletion docs/Architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ flowchart LR
Operations --> SDK["Azure Blob SDK"]
```

For immutable uploads, `WriteIfAbsentOrSameAsync` streams an expected-length request through SHA-256, retries a conflicting write by reading the current ETag-pinned object, and accepts only byte-identical content. It returns the stored ETag, digest, and whether the object was reused; mismatched content keeps the provider conflict. Neither input nor stored content is buffered as a whole.
For immutable uploads, `WriteIfAbsentOrSameAsync` streams an expected-length request through SHA-256, retries a conflicting write by reading the current ETag-pinned object, and accepts only byte-identical content with matching content type, encoding, and metadata. It returns the stored ETag, digest, and whether the object was reused; a mismatch keeps the provider conflict. Neither input nor stored content is buffered as a whole.

Object conditions (`IfAbsent`, `IfMatch`) are enforced by the service for writes,
metadata updates and reads. Read ranges use a fixed ETag and stream data without
Expand Down
Loading