Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,20 @@
import com.google.protobuf.Descriptors.DescriptorValidationException;
import java.io.IOException;
import java.util.concurrent.Phaser;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import javax.annotation.concurrent.GuardedBy;

class BigQueryJdbcBulkInsertWriter {
private static final BigQueryJdbcCustomLogger LOG =
new BigQueryJdbcCustomLogger(BigQueryJdbcBulkInsertWriter.class.getName());
private JsonStreamWriter jsonStreamWriter;
JsonStreamWriter jsonStreamWriter;
private volatile String streamName;
private final Phaser openRequestCount = new Phaser(1);
private final AtomicBoolean writerClosed = new AtomicBoolean(false);
private final Object streamLock = new Object();

@GuardedBy("lock")
@GuardedBy("streamLock")
private RuntimeException error = null;

void initialize(TableName parentTable, BigQueryWriteClient client, RetrySettings retrySettings)
Expand All @@ -59,15 +62,21 @@ void initialize(TableName parentTable, BigQueryWriteClient client, RetrySettings
.setWriteStream(stream)
.build();
WriteStream writeStream = client.createWriteStream(createWriteStreamRequest);
this.streamName = writeStream.getName();

JsonStreamWriter.Builder jsonStreamWriterBuilder =
JsonStreamWriter.newBuilder(writeStream.getName(), writeStream.getTableSchema(), client);
JsonStreamWriter.newBuilder(this.streamName, writeStream.getTableSchema(), client);

if (retrySettings != null) {
jsonStreamWriterBuilder.setRetrySettings(retrySettings);
}

this.jsonStreamWriter = jsonStreamWriterBuilder.build();
this.jsonStreamWriter = buildJsonStreamWriter(jsonStreamWriterBuilder);
}

JsonStreamWriter buildJsonStreamWriter(JsonStreamWriter.Builder builder)
throws DescriptorValidationException, IOException, InterruptedException {
return builder.build();
}

void append(JsonArray data, long offset) throws DescriptorValidationException, IOException {
Expand All @@ -80,62 +89,98 @@ void append(JsonArray data, long offset) throws DescriptorValidationException, I
}
}

ApiFuture<AppendRowsResponse> future = jsonStreamWriter.append(data, offset);
ApiFutures.addCallback(
future, new AppendCompleteCallback(this), MoreExecutors.directExecutor());
// Register before submitting so a synchronously completed future cannot drop the party count
// to zero (which permanently terminates the Phaser) before registration occurs.
openRequestCount.register();
AppendCompleteCallback callback = new AppendCompleteCallback(this);
try {
ApiFuture<AppendRowsResponse> future = jsonStreamWriter.append(data, offset);
ApiFutures.addCallback(future, callback, MoreExecutors.directExecutor());
} catch (Throwable t) {
callback.done();
throw t;
}
}

long cleanup(BigQueryWriteClient client) {
long cleanup(BigQueryWriteClient client) throws InterruptedException {
LOG.finer("++enter++");
LOG.fine("Cleaning up bulk insert writer for stream: %s", jsonStreamWriter.getStreamName());
openRequestCount.arriveAndAwaitAdvance();
jsonStreamWriter.close();
String currentStreamName = getStreamName();
LOG.fine("Cleaning up bulk insert writer for stream: %s", currentStreamName);
try {
openRequestCount.awaitAdvanceInterruptibly(openRequestCount.arrive());
} finally {
closeStreamWriter();
}

synchronized (this.streamLock) {
if (this.error != null) {
throw this.error;
}
}

// Finalize the stream.
FinalizeWriteStreamResponse finalizeResponse =
client.finalizeWriteStream(jsonStreamWriter.getStreamName());
LOG.finer("Rows written: " + finalizeResponse.getRowCount());
FinalizeWriteStreamResponse finalizeResponse = client.finalizeWriteStream(currentStreamName);
LOG.finer("Rows written: %d", finalizeResponse.getRowCount());
return finalizeResponse.getRowCount();
}

void closeQuietly() {
if (this.jsonStreamWriter != null && this.writerClosed.compareAndSet(false, true)) {
try {
this.jsonStreamWriter.close();
} catch (RuntimeException ex) {
LOG.warning(ex, "Failed to close JsonStreamWriter during error cleanup.");
}
}
}

private void closeStreamWriter() {
if (this.jsonStreamWriter != null && this.writerClosed.compareAndSet(false, true)) {
this.jsonStreamWriter.close();
}
}

String getStreamName() {
return jsonStreamWriter.getStreamName();
if (this.streamName != null) {
return this.streamName;
}
return this.jsonStreamWriter != null ? this.jsonStreamWriter.getStreamName() : null;
}

static class AppendCompleteCallback implements ApiFutureCallback<AppendRowsResponse> {

private final BigQueryJdbcBulkInsertWriter parent;
private final AtomicBoolean deregistered = new AtomicBoolean(false);

AppendCompleteCallback(BigQueryJdbcBulkInsertWriter parent) {
this.parent = parent;
}

@Override
public void onSuccess(AppendRowsResponse response) {
done();
}

@Override
public void onFailure(Throwable throwable) {
parent.LOG.warning(
"Append failed for stream %s: %s", parent.getStreamName(), throwable.getMessage());
synchronized (this.parent.streamLock) {
if (this.parent.error == null) {
StorageException storageException = Exceptions.toStorageException(throwable);
this.parent.error =
(storageException != null) ? storageException : new RuntimeException(throwable);
try {
synchronized (this.parent.streamLock) {
if (this.parent.error == null) {
StorageException storageException = Exceptions.toStorageException(throwable);
this.parent.error =
(storageException != null) ? storageException : new RuntimeException(throwable);
}
}
LOG.warning(
"Append failed for stream %s: %s", parent.getStreamName(), throwable.getMessage());
} finally {
done();
}
done();
}

private void done() {
this.parent.openRequestCount.arriveAndDeregister();
void done() {
if (this.deregistered.compareAndSet(false, true)) {
this.parent.openRequestCount.arriveAndDeregister();
}
}
}
}
Loading
Loading