Add content hasher utility

core/api/core.api

@@ -827,6 +827,14 @@ public class com/dropbox/core/util/CountingOutputStream : java/io/OutputStream {
 	public fun write ([BII)V
 }
 
+public final class com/dropbox/core/util/DropboxContentHasher {
+	public static final field BLOCK_SIZE I
+	public static fun hash (Ljava/io/InputStream;)[B
+	public static fun hash ([B)[B
+	public static fun hashHex (Ljava/io/InputStream;)Ljava/lang/String;
+	public static fun hashHex ([B)Ljava/lang/String;
+}
+
 public abstract class com/dropbox/core/util/DumpWriter {
 	public fun <init> ()V
 	public abstract fun f (Ljava/lang/String;)Lcom/dropbox/core/util/DumpWriter;

core/src/main/java/com/dropbox/core/util/DropboxContentHasher.java

@@ -0,0 +1,141 @@
+package com.dropbox.core.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Objects;
+
+/**
+ * Utility for calculating Dropbox content hashes.
+ *
+ * <p>The Dropbox content hash is calculated by splitting the input into
+ * 4 MiB blocks, calculating the SHA-256 hash of each block, concatenating
+ * those block hashes, and calculating the SHA-256 hash of the resulting
+ * sequence.</p>
+ */
+public final class DropboxContentHasher {
+    public static final int BLOCK_SIZE = 4 * 1024 * 1024;
+
+    private static final String SHA_256 = "SHA-256";
+    private static final char[] HEX_DIGITS = "0123456789abcdef".toCharArray();
+
+    private DropboxContentHasher() {
+    }
+
+    /**
+     * Calculates the Dropbox content hash for an input stream.
+     *
+     * <p>This method does not close the supplied stream.</p>
+     *
+     * @param input stream to read
+     * @return raw 32-byte Dropbox content hash
+     * @throws IOException if the stream cannot be read
+     */
+    public static byte[] hash(InputStream input) throws IOException {
+        Objects.requireNonNull(input, "input");
+
+        MessageDigest overallDigest = newSha256Digest();
+        MessageDigest blockDigest = newSha256Digest();
+        byte[] buffer = new byte[BLOCK_SIZE];
+
+        while (true) {
+            int blockLength = readBlock(input, buffer);
+            if (blockLength == 0) {
+                break;
+            }
+
+            blockDigest.reset();
+            blockDigest.update(buffer, 0, blockLength);
+            overallDigest.update(blockDigest.digest());
+        }
+
+        return overallDigest.digest();
+    }
+
+    /**
+     * Calculates the lowercase hexadecimal Dropbox content hash for an input stream.
+     *
+     * <p>This method does not close the supplied stream.</p>
+     *
+     * @param input stream to read
+     * @return lowercase hexadecimal Dropbox content hash
+     * @throws IOException if the stream cannot be read
+     */
+    public static String hashHex(InputStream input) throws IOException {
+        return toHex(hash(input));
+    }
+
+    /**
+     * Calculates the Dropbox content hash for a byte array.
+     *
+     * @param input bytes to hash
+     * @return raw 32-byte Dropbox content hash
+     */
+    public static byte[] hash(byte[] input) {
+        Objects.requireNonNull(input, "input");
+
+        try {
+            return hash(new ByteArrayInputStream(input));
+        } catch (IOException ex) {
+            // ByteArrayInputStream does not throw IOException while reading.
+            throw new AssertionError("Unexpected ByteArrayInputStream failure", ex);
+        }
+    }
+
+    /**
+     * Calculates the lowercase hexadecimal Dropbox content hash for a byte array.
+     *
+     * @param input bytes to hash
+     * @return lowercase hexadecimal Dropbox content hash
+     */
+    public static String hashHex(byte[] input) {
+        return toHex(hash(input));
+    }
+
+    private static int readBlock(InputStream input, byte[] buffer) throws IOException {
+        int offset = 0;
+
+        while (offset < buffer.length) {
+            int bytesRead = input.read(buffer, offset, buffer.length - offset);
+
+            if (bytesRead < 0) {
+                break;
+            }
+
+            if (bytesRead == 0) {
+                int value = input.read();
+                if (value < 0) {
+                    break;
+                }
+
+                buffer[offset++] = (byte) value;
+            } else {
+                offset += bytesRead;
+            }
+        }
+
+        return offset;
+    }
+
+    private static MessageDigest newSha256Digest() {
+        try {
+            return MessageDigest.getInstance(SHA_256);
+        } catch (NoSuchAlgorithmException ex) {
+            throw new IllegalStateException("SHA-256 is not available", ex);
+        }
+    }
+
+    private static String toHex(byte[] bytes) {
+        char[] result = new char[bytes.length * 2];
+
+        for (int i = 0; i < bytes.length; i++) {
+            int value = bytes[i] & 0xff;
+            result[i * 2] = HEX_DIGITS[value >>> 4];
+            result[i * 2 + 1] = HEX_DIGITS[value & 0x0f];
+        }
+
+        return new String(result);
+    }
+}

core/src/test/java/com/dropbox/core/util/DropboxContentHasherTest.java

@@ -0,0 +1,115 @@
+package com.dropbox.core.util;
+
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Random;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+
+public class DropboxContentHasherTest {
+
+    @Test
+    public void testEmpty() {
+        assertEquals(
+                DropboxContentHasher.hashHex(new byte[0]),
+                "e3b0c44298fc1c149afbf4c8996fb924"
+                        + "27ae41e4649b934ca495991b7852b855");
+    }
+
+    @Test
+    public void testByteArrayAndStreamMatch() throws Exception {
+        byte[] data = randomBytes(12345);
+
+        assertEquals(
+                DropboxContentHasher.hashHex(data),
+                DropboxContentHasher.hashHex(new ByteArrayInputStream(data))
+        );
+    }
+
+    @Test
+    public void testExactlyOneBlock() throws Exception {
+        byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE);
+
+        assertEquals(
+                DropboxContentHasher.hashHex(data),
+                DropboxContentHasher.hashHex(new ByteArrayInputStream(data))
+        );
+    }
+
+    @Test
+    public void testBlockPlusOneByte() throws Exception {
+        byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE + 1);
+
+        assertEquals(
+                DropboxContentHasher.hashHex(data),
+                DropboxContentHasher.hashHex(new ByteArrayInputStream(data))
+        );
+    }
+
+    @Test
+    public void testMultipleBlocks() throws Exception {
+        byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE * 3 + 123);
+
+        assertEquals(
+                DropboxContentHasher.hashHex(data),
+                DropboxContentHasher.hashHex(new ByteArrayInputStream(data))
+        );
+    }
+
+    @Test
+    public void testPartialReads() throws Exception {
+        byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE + 123);
+
+        InputStream partial = new FilterInputStream(
+                new ByteArrayInputStream(data)
+        ) {
+            @Override
+            public int read(byte[] b, int off, int len) throws IOException {
+                return super.read(b, off, Math.min(len, 37));
+            }
+        };
+
+        assertEquals(
+                DropboxContentHasher.hashHex(data),
+                DropboxContentHasher.hashHex(partial)
+        );
+    }
+
+    @Test
+    public void testDoesNotCloseStream() throws Exception {
+        CloseTrackingInputStream in =
+                new CloseTrackingInputStream(
+                        new ByteArrayInputStream(randomBytes(1024)));
+
+        DropboxContentHasher.hashHex(in);
+
+        assertFalse(in.closed);
+    }
+
+    private static byte[] randomBytes(int size) {
+        byte[] data = new byte[size];
+        new Random(12345).nextBytes(data);
+        return data;
+    }
+
+    private static final class CloseTrackingInputStream
+            extends FilterInputStream {
+
+        boolean closed;
+
+        CloseTrackingInputStream(InputStream in) {
+            super(in);
+        }
+
+        @Override
+        public void close() throws IOException {
+            closed = true;
+            super.close();
+        }
+    }
+}

core/src/test/java/com/dropbox/core/v2/DbxClientV2IT.java

@@ -1,30 +1,25 @@
 package com.dropbox.core.v2;
 
-import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth.assertWithMessage;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.util.Date;
-import java.util.List;
-import java.util.Locale;
-
-import com.dropbox.core.util.IOUtil.ProgressListener;
-import org.testng.annotations.Test;
-
 import com.dropbox.core.BadRequestException;
 import com.dropbox.core.DbxApiException;
 import com.dropbox.core.DbxDownloader;
 import com.dropbox.core.ITUtil;
+import com.dropbox.core.util.DropboxContentHasher;
+import com.dropbox.core.util.IOUtil.ProgressListener;
 import com.dropbox.core.v2.fileproperties.PropertyGroup;
-import com.dropbox.core.v2.files.FileMetadata;
-import com.dropbox.core.v2.files.GetMetadataError;
-import com.dropbox.core.v2.files.GetMetadataErrorException;
-import com.dropbox.core.v2.files.LookupError;
-import com.dropbox.core.v2.files.Metadata;
-import com.dropbox.core.v2.files.WriteMode;
+import com.dropbox.core.v2.files.*;
 import com.dropbox.core.v2.users.BasicAccount;
 import com.dropbox.core.v2.users.FullAccount;
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
 
 public class DbxClientV2IT {
     @Test
@@ -99,6 +94,7 @@ private void testUploadAndDownload(DbxClientV2 client, boolean trackProgress) th
         assertThat(metadata.getName()).isEqualTo(filename);
         assertThat(metadata.getPathLower()).isEqualTo(path.toLowerCase());
         assertThat(metadata.getSize()).isEqualTo(contents.length);
+        assertThat(metadata.getContentHash()).isEqualTo(DropboxContentHasher.hashHex(contents));
 
         Metadata actual = client.files().getMetadata(path);
 
@@ -119,6 +115,7 @@ private void testUploadAndDownload(DbxClientV2 client, boolean trackProgress) th
             assertFileMetadataEquivalent(actualResult, metadata);
             assertThat(actualContents).isEqualTo(contents);
             assertThat(downloader.getContentType()).isEqualTo("application/octet-stream");
+            assertThat(DropboxContentHasher.hashHex(actualContents)).isEqualTo(metadata.getContentHash());
         } catch (AssertionError e) {
             // so subsequent tests don't fail due to file not being cleaned up
             client.files().deleteV2(path).getMetadata();