diff --git a/core/api/core.api b/core/api/core.api index 5d736172ce..780f394934 100644 --- a/core/api/core.api +++ b/core/api/core.api @@ -827,6 +827,14 @@ public class com/dropbox/core/util/CountingOutputStream : java/io/OutputStream { public fun write ([BII)V } +public final class com/dropbox/core/util/DropboxContentHasher { + public static final field BLOCK_SIZE I + public static fun hash (Ljava/io/InputStream;)[B + public static fun hash ([B)[B + public static fun hashHex (Ljava/io/InputStream;)Ljava/lang/String; + public static fun hashHex ([B)Ljava/lang/String; +} + public abstract class com/dropbox/core/util/DumpWriter { public fun ()V public abstract fun f (Ljava/lang/String;)Lcom/dropbox/core/util/DumpWriter; diff --git a/core/src/main/java/com/dropbox/core/util/DropboxContentHasher.java b/core/src/main/java/com/dropbox/core/util/DropboxContentHasher.java new file mode 100644 index 0000000000..b9b01ae6d6 --- /dev/null +++ b/core/src/main/java/com/dropbox/core/util/DropboxContentHasher.java @@ -0,0 +1,141 @@ +package com.dropbox.core.util; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Objects; + +/** + * Utility for calculating Dropbox content hashes. + * + *

The Dropbox content hash is calculated by splitting the input into + * 4 MiB blocks, calculating the SHA-256 hash of each block, concatenating + * those block hashes, and calculating the SHA-256 hash of the resulting + * sequence.

+ */ +public final class DropboxContentHasher { + public static final int BLOCK_SIZE = 4 * 1024 * 1024; + + private static final String SHA_256 = "SHA-256"; + private static final char[] HEX_DIGITS = "0123456789abcdef".toCharArray(); + + private DropboxContentHasher() { + } + + /** + * Calculates the Dropbox content hash for an input stream. + * + *

This method does not close the supplied stream.

+ * + * @param input stream to read + * @return raw 32-byte Dropbox content hash + * @throws IOException if the stream cannot be read + */ + public static byte[] hash(InputStream input) throws IOException { + Objects.requireNonNull(input, "input"); + + MessageDigest overallDigest = newSha256Digest(); + MessageDigest blockDigest = newSha256Digest(); + byte[] buffer = new byte[BLOCK_SIZE]; + + while (true) { + int blockLength = readBlock(input, buffer); + if (blockLength == 0) { + break; + } + + blockDigest.reset(); + blockDigest.update(buffer, 0, blockLength); + overallDigest.update(blockDigest.digest()); + } + + return overallDigest.digest(); + } + + /** + * Calculates the lowercase hexadecimal Dropbox content hash for an input stream. + * + *

This method does not close the supplied stream.

+ * + * @param input stream to read + * @return lowercase hexadecimal Dropbox content hash + * @throws IOException if the stream cannot be read + */ + public static String hashHex(InputStream input) throws IOException { + return toHex(hash(input)); + } + + /** + * Calculates the Dropbox content hash for a byte array. + * + * @param input bytes to hash + * @return raw 32-byte Dropbox content hash + */ + public static byte[] hash(byte[] input) { + Objects.requireNonNull(input, "input"); + + try { + return hash(new ByteArrayInputStream(input)); + } catch (IOException ex) { + // ByteArrayInputStream does not throw IOException while reading. + throw new AssertionError("Unexpected ByteArrayInputStream failure", ex); + } + } + + /** + * Calculates the lowercase hexadecimal Dropbox content hash for a byte array. + * + * @param input bytes to hash + * @return lowercase hexadecimal Dropbox content hash + */ + public static String hashHex(byte[] input) { + return toHex(hash(input)); + } + + private static int readBlock(InputStream input, byte[] buffer) throws IOException { + int offset = 0; + + while (offset < buffer.length) { + int bytesRead = input.read(buffer, offset, buffer.length - offset); + + if (bytesRead < 0) { + break; + } + + if (bytesRead == 0) { + int value = input.read(); + if (value < 0) { + break; + } + + buffer[offset++] = (byte) value; + } else { + offset += bytesRead; + } + } + + return offset; + } + + private static MessageDigest newSha256Digest() { + try { + return MessageDigest.getInstance(SHA_256); + } catch (NoSuchAlgorithmException ex) { + throw new IllegalStateException("SHA-256 is not available", ex); + } + } + + private static String toHex(byte[] bytes) { + char[] result = new char[bytes.length * 2]; + + for (int i = 0; i < bytes.length; i++) { + int value = bytes[i] & 0xff; + result[i * 2] = HEX_DIGITS[value >>> 4]; + result[i * 2 + 1] = HEX_DIGITS[value & 0x0f]; + } + + return new String(result); + } +} diff --git a/core/src/test/java/com/dropbox/core/util/DropboxContentHasherTest.java b/core/src/test/java/com/dropbox/core/util/DropboxContentHasherTest.java new file mode 100644 index 0000000000..e256b8527d --- /dev/null +++ b/core/src/test/java/com/dropbox/core/util/DropboxContentHasherTest.java @@ -0,0 +1,115 @@ +package com.dropbox.core.util; + +import org.testng.annotations.Test; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Random; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; + +public class DropboxContentHasherTest { + + @Test + public void testEmpty() { + assertEquals( + DropboxContentHasher.hashHex(new byte[0]), + "e3b0c44298fc1c149afbf4c8996fb924" + + "27ae41e4649b934ca495991b7852b855"); + } + + @Test + public void testByteArrayAndStreamMatch() throws Exception { + byte[] data = randomBytes(12345); + + assertEquals( + DropboxContentHasher.hashHex(data), + DropboxContentHasher.hashHex(new ByteArrayInputStream(data)) + ); + } + + @Test + public void testExactlyOneBlock() throws Exception { + byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE); + + assertEquals( + DropboxContentHasher.hashHex(data), + DropboxContentHasher.hashHex(new ByteArrayInputStream(data)) + ); + } + + @Test + public void testBlockPlusOneByte() throws Exception { + byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE + 1); + + assertEquals( + DropboxContentHasher.hashHex(data), + DropboxContentHasher.hashHex(new ByteArrayInputStream(data)) + ); + } + + @Test + public void testMultipleBlocks() throws Exception { + byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE * 3 + 123); + + assertEquals( + DropboxContentHasher.hashHex(data), + DropboxContentHasher.hashHex(new ByteArrayInputStream(data)) + ); + } + + @Test + public void testPartialReads() throws Exception { + byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE + 123); + + InputStream partial = new FilterInputStream( + new ByteArrayInputStream(data) + ) { + @Override + public int read(byte[] b, int off, int len) throws IOException { + return super.read(b, off, Math.min(len, 37)); + } + }; + + assertEquals( + DropboxContentHasher.hashHex(data), + DropboxContentHasher.hashHex(partial) + ); + } + + @Test + public void testDoesNotCloseStream() throws Exception { + CloseTrackingInputStream in = + new CloseTrackingInputStream( + new ByteArrayInputStream(randomBytes(1024))); + + DropboxContentHasher.hashHex(in); + + assertFalse(in.closed); + } + + private static byte[] randomBytes(int size) { + byte[] data = new byte[size]; + new Random(12345).nextBytes(data); + return data; + } + + private static final class CloseTrackingInputStream + extends FilterInputStream { + + boolean closed; + + CloseTrackingInputStream(InputStream in) { + super(in); + } + + @Override + public void close() throws IOException { + closed = true; + super.close(); + } + } +} diff --git a/core/src/test/java/com/dropbox/core/v2/DbxClientV2IT.java b/core/src/test/java/com/dropbox/core/v2/DbxClientV2IT.java index af5df1d982..b997c44d0c 100644 --- a/core/src/test/java/com/dropbox/core/v2/DbxClientV2IT.java +++ b/core/src/test/java/com/dropbox/core/v2/DbxClientV2IT.java @@ -1,30 +1,25 @@ package com.dropbox.core.v2; -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.Date; -import java.util.List; -import java.util.Locale; - -import com.dropbox.core.util.IOUtil.ProgressListener; -import org.testng.annotations.Test; - import com.dropbox.core.BadRequestException; import com.dropbox.core.DbxApiException; import com.dropbox.core.DbxDownloader; import com.dropbox.core.ITUtil; +import com.dropbox.core.util.DropboxContentHasher; +import com.dropbox.core.util.IOUtil.ProgressListener; import com.dropbox.core.v2.fileproperties.PropertyGroup; -import com.dropbox.core.v2.files.FileMetadata; -import com.dropbox.core.v2.files.GetMetadataError; -import com.dropbox.core.v2.files.GetMetadataErrorException; -import com.dropbox.core.v2.files.LookupError; -import com.dropbox.core.v2.files.Metadata; -import com.dropbox.core.v2.files.WriteMode; +import com.dropbox.core.v2.files.*; import com.dropbox.core.v2.users.BasicAccount; import com.dropbox.core.v2.users.FullAccount; +import org.testng.annotations.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.Date; +import java.util.List; +import java.util.Locale; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; public class DbxClientV2IT { @Test @@ -99,6 +94,7 @@ private void testUploadAndDownload(DbxClientV2 client, boolean trackProgress) th assertThat(metadata.getName()).isEqualTo(filename); assertThat(metadata.getPathLower()).isEqualTo(path.toLowerCase()); assertThat(metadata.getSize()).isEqualTo(contents.length); + assertThat(metadata.getContentHash()).isEqualTo(DropboxContentHasher.hashHex(contents)); Metadata actual = client.files().getMetadata(path); @@ -119,6 +115,7 @@ private void testUploadAndDownload(DbxClientV2 client, boolean trackProgress) th assertFileMetadataEquivalent(actualResult, metadata); assertThat(actualContents).isEqualTo(contents); assertThat(downloader.getContentType()).isEqualTo("application/octet-stream"); + assertThat(DropboxContentHasher.hashHex(actualContents)).isEqualTo(metadata.getContentHash()); } catch (AssertionError e) { // so subsequent tests don't fail due to file not being cleaned up client.files().deleteV2(path).getMetadata();