Skip to content

Commit eb5d77a

Browse files
BenWhiteheadBenWhitehead
committed
feat: implement BufferToDiskThenUpload BlobWriteSessionConfig
There are scenarios in which disk space is more plentiful than memory space. This new BlobWriteSessionConfig allows augmenting an instance of storage to prefer buffering to disk rather than keeping things in memory. Once the file on disk is closed, the entire file will then be uploaded to GCS.
1 parent d277957 commit eb5d77a

File tree

10 files changed

+746
-3
lines changed

10 files changed

+746
-3
lines changed

‎google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteSessionConfigs.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919
import com.google.api.core.BetaApi;
2020
import com.google.cloud.storage.GrpcStorageOptions.GrpcStorageDefaults;
2121
import com.google.cloud.storage.Storage.BlobWriteOption;
22+
import com.google.common.collect.ImmutableList;
23+
import java.io.IOException;
24+
import java.nio.file.Path;
25+
import java.nio.file.Paths;
26+
import java.util.Collection;
2227

2328
/**
2429
* Factory class to select and construct {@link BlobWriteSessionConfig}s.
@@ -46,4 +51,54 @@ private BlobWriteSessionConfigs() {}
4651
public static DefaultBlobWriteSessionConfig getDefault() {
4752
return new DefaultBlobWriteSessionConfig(ByteSizeConstants._16MiB);
4853
}
54+
55+
/**
56+
* Create a new {@link BlobWriteSessionConfig} which will first buffer the content of the object
57+
* to a temporary file under {@code java.io.tmpdir}.
58+
*
59+
* <p>Once the file on disk is closed, the entire file will then be uploaded to Google Cloud
60+
* Storage.
61+
*
62+
* @see Storage#blobWriteSession(BlobInfo, BlobWriteOption...)
63+
* @see GrpcStorageOptions.Builder#setBlobWriteSessionConfig(BlobWriteSessionConfig)
64+
*/
65+
@BetaApi
66+
public static BlobWriteSessionConfig bufferToTempDirThenUpload() throws IOException {
67+
return bufferToDiskThenUpload(
68+
Paths.get(System.getProperty("java.io.tmpdir"), "google-cloud-storage"));
69+
}
70+
71+
/**
72+
* Create a new {@link BlobWriteSessionConfig} which will first buffer the content of the object
73+
* to a temporary file under the specified {@code path}.
74+
*
75+
* <p>Once the file on disk is closed, the entire file will then be uploaded to Google Cloud
76+
* Storage.
77+
*
78+
* @see Storage#blobWriteSession(BlobInfo, BlobWriteOption...)
79+
* @see GrpcStorageOptions.Builder#setBlobWriteSessionConfig(BlobWriteSessionConfig)
80+
*/
81+
@BetaApi
82+
public static BufferToDiskThenUpload bufferToDiskThenUpload(Path path) throws IOException {
83+
return bufferToDiskThenUpload(ImmutableList.of(path));
84+
}
85+
86+
/**
87+
* Create a new {@link BlobWriteSessionConfig} which will first buffer the content of the object
88+
* to a temporary file under one of the specified {@code paths}.
89+
*
90+
* <p>Once the file on disk is closed, the entire file will then be uploaded to Google Cloud
91+
* Storage.
92+
*
93+
* <p>The specifics of how the work is spread across multiple paths is undefined and subject to
94+
* change.
95+
*
96+
* @see Storage#blobWriteSession(BlobInfo, BlobWriteOption...)
97+
* @see GrpcStorageOptions.Builder#setBlobWriteSessionConfig(BlobWriteSessionConfig)
98+
*/
99+
@BetaApi
100+
public static BufferToDiskThenUpload bufferToDiskThenUpload(Collection<Path> paths)
101+
throws IOException {
102+
return new BufferToDiskThenUpload(ImmutableList.copyOf(paths), false);
103+
}
49104
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/BufferToDiskThenUpload.java‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.core.ApiFuture;
20+
import com.google.api.core.ApiFutures;
21+
import com.google.api.core.BetaApi;
22+
import com.google.api.core.InternalApi;
23+
import com.google.api.core.SettableApiFuture;
24+
import com.google.cloud.storage.Conversions.Decoder;
25+
import com.google.cloud.storage.Storage.BlobWriteOption;
26+
import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt;
27+
import com.google.cloud.storage.UnifiedOpts.Opts;
28+
import com.google.common.annotations.VisibleForTesting;
29+
import com.google.common.collect.ImmutableList;
30+
import com.google.common.util.concurrent.MoreExecutors;
31+
import com.google.storage.v2.WriteObjectResponse;
32+
import java.io.IOException;
33+
import java.nio.ByteBuffer;
34+
import java.nio.channels.WritableByteChannel;
35+
import java.nio.file.Files;
36+
import java.nio.file.Path;
37+
import java.time.Clock;
38+
import java.time.Duration;
39+
import java.util.Collection;
40+
import javax.annotation.concurrent.Immutable;
41+
42+
/**
43+
* There are scenarios in which disk space is more plentiful than memory space. This new {@link
44+
* BlobWriteSessionConfig} allows augmenting an instance of storage to produce {@link
45+
* BlobWriteSession}s which will buffer to disk rather than holding things in memory.
46+
*
47+
* <p>Once the file on disk is closed, the entire file will then be uploaded to GCS.
48+
*
49+
* @see Storage#blobWriteSession(BlobInfo, BlobWriteOption...)
50+
* @see GrpcStorageOptions.Builder#setBlobWriteSessionConfig(BlobWriteSessionConfig)
51+
* @see BlobWriteSessionConfigs#bufferToDiskThenUpload(Path)
52+
* @see BlobWriteSessionConfigs#bufferToDiskThenUpload(Collection)
53+
*/
54+
@Immutable
55+
@BetaApi
56+
public final class BufferToDiskThenUpload extends BlobWriteSessionConfig {
57+
58+
private final ImmutableList<Path> paths;
59+
private final boolean includeLoggingSink;
60+
61+
@InternalApi
62+
BufferToDiskThenUpload(ImmutableList<Path> paths, boolean includeLoggingSink) throws IOException {
63+
this.paths = paths;
64+
this.includeLoggingSink = includeLoggingSink;
65+
}
66+
67+
@VisibleForTesting
68+
@InternalApi
69+
BufferToDiskThenUpload withIncludeLoggingSink() throws IOException {
70+
return new BufferToDiskThenUpload(paths, true);
71+
}
72+
73+
@InternalApi
74+
@Override
75+
WriterFactory createFactory(Clock clock) throws IOException {
76+
Duration window = Duration.ofMinutes(10);
77+
RecoveryFileManager recoveryFileManager =
78+
RecoveryFileManager.of(paths, getRecoverVolumeSinkFactory(clock, window));
79+
ThroughputSink gcs = ThroughputSink.windowed(ThroughputMovingWindow.of(window), clock);
80+
gcs = includeLoggingSink ? ThroughputSink.tee(ThroughputSink.logged("gcs", clock), gcs) : gcs;
81+
return new Factory(recoveryFileManager, clock, gcs);
82+
}
83+
84+
private RecoveryFileManager.RecoverVolumeSinkFactory getRecoverVolumeSinkFactory(
85+
Clock clock, Duration window) {
86+
return path -> {
87+
ThroughputSink windowed = ThroughputSink.windowed(ThroughputMovingWindow.of(window), clock);
88+
if (includeLoggingSink) {
89+
return ThroughputSink.tee(
90+
ThroughputSink.logged(path.toAbsolutePath().toString(), clock), windowed);
91+
} else {
92+
return windowed;
93+
}
94+
};
95+
}
96+
97+
private static final class Factory implements WriterFactory {
98+
99+
private final RecoveryFileManager recoveryFileManager;
100+
private final Clock clock;
101+
private final ThroughputSink gcs;
102+
103+
private Factory(RecoveryFileManager recoveryFileManager, Clock clock, ThroughputSink gcs) {
104+
this.recoveryFileManager = recoveryFileManager;
105+
this.clock = clock;
106+
this.gcs = gcs;
107+
}
108+
109+
@InternalApi
110+
@Override
111+
public WritableByteChannelSession<?, BlobInfo> writeSession(
112+
StorageInternal storage,
113+
BlobInfo info,
114+
Opts<ObjectTargetOpt> opts,
115+
Decoder<WriteObjectResponse, BlobInfo> d) {
116+
return new Factory.WriteToFileThenUpload(
117+
storage, info, opts, recoveryFileManager.newRecoveryFile(info));
118+
}
119+
120+
private final class WriteToFileThenUpload
121+
implements WritableByteChannelSession<WritableByteChannel, BlobInfo> {
122+
123+
private final StorageInternal storage;
124+
private final BlobInfo info;
125+
private final Opts<ObjectTargetOpt> opts;
126+
private final RecoveryFile rf;
127+
private final SettableApiFuture<BlobInfo> result;
128+
129+
private WriteToFileThenUpload(
130+
StorageInternal storage, BlobInfo info, Opts<ObjectTargetOpt> opts, RecoveryFile rf) {
131+
this.info = info;
132+
this.opts = opts;
133+
this.rf = rf;
134+
this.storage = storage;
135+
this.result = SettableApiFuture.create();
136+
}
137+
138+
@Override
139+
public ApiFuture<WritableByteChannel> openAsync() {
140+
try {
141+
ApiFuture<WritableByteChannel> f = ApiFutures.immediateFuture(rf.writer());
142+
return ApiFutures.transform(
143+
f, Factory.WriteToFileThenUpload.Flusher::new, MoreExecutors.directExecutor());
144+
} catch (IOException e) {
145+
throw StorageException.coalesce(e);
146+
}
147+
}
148+
149+
@Override
150+
public ApiFuture<BlobInfo> getResult() {
151+
return result;
152+
}
153+
154+
private final class Flusher implements WritableByteChannel {
155+
156+
private final WritableByteChannel delegate;
157+
158+
private Flusher(WritableByteChannel delegate) {
159+
this.delegate = delegate;
160+
}
161+
162+
@Override
163+
public int write(ByteBuffer src) throws IOException {
164+
return delegate.write(src);
165+
}
166+
167+
@Override
168+
public boolean isOpen() {
169+
return delegate.isOpen();
170+
}
171+
172+
@Override
173+
public void close() throws IOException {
174+
delegate.close();
175+
try (RecoveryFile rf = Factory.WriteToFileThenUpload.this.rf) {
176+
Path path = rf.getPath();
177+
long size = Files.size(path);
178+
ThroughputSink.computeThroughput(
179+
clock,
180+
gcs,
181+
size,
182+
() -> {
183+
BlobInfo blob = storage.internalCreateFrom(path, info, opts);
184+
result.set(blob);
185+
});
186+
} catch (StorageException | IOException e) {
187+
result.setException(e);
188+
throw e;
189+
}
190+
}
191+
}
192+
}
193+
}
194+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,8 @@
152152
import org.checkerframework.checker.nullness.qual.Nullable;
153153

154154
@BetaApi
155-
final class GrpcStorageImpl extends BaseService<StorageOptions> implements StorageInternal {
155+
final class GrpcStorageImpl extends BaseService<StorageOptions>
156+
implements Storage, StorageInternal {
156157

157158
private static final byte[] ZERO_BYTES = new byte[0];
158159
private static final Set<OpenOption> READ_OPS = ImmutableSet.of(StandardOpenOption.READ);

‎google-cloud-storage/src/main/java/com/google/cloud/storage/RecoveryFile.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.common.base.MoreObjects;
20+
import com.google.common.collect.ImmutableSet;
21+
import java.io.IOException;
22+
import java.nio.channels.SeekableByteChannel;
23+
import java.nio.channels.WritableByteChannel;
24+
import java.nio.file.Files;
25+
import java.nio.file.OpenOption;
26+
import java.nio.file.Path;
27+
import java.nio.file.StandardOpenOption;
28+
import java.util.Set;
29+
30+
/**
31+
* When uploading to GCS, there are times when memory buffers are not preferable. This class
32+
* encapsulates the logic and lifecycle for a file written to local disk which can be used for
33+
* upload recovery in the case an upload is interrupted.
34+
*/
35+
final class RecoveryFile implements AutoCloseable {
36+
private static final Set<OpenOption> writeOps =
37+
ImmutableSet.of(StandardOpenOption.CREATE, StandardOpenOption.WRITE);
38+
private static final Set<OpenOption> readOps = ImmutableSet.of(StandardOpenOption.READ);
39+
40+
private final Path path;
41+
private final ThroughputSink throughputSink;
42+
private final Runnable onCloseCallback;
43+
44+
RecoveryFile(Path path, ThroughputSink throughputSink, Runnable onCloseCallback) {
45+
this.path = path;
46+
this.throughputSink = throughputSink;
47+
this.onCloseCallback = onCloseCallback;
48+
}
49+
50+
public Path getPath() {
51+
return path;
52+
}
53+
54+
public Path touch() throws IOException {
55+
return Files.createFile(path);
56+
}
57+
58+
public SeekableByteChannel reader() throws IOException {
59+
return Files.newByteChannel(path, readOps);
60+
}
61+
62+
public WritableByteChannel writer() throws IOException {
63+
return throughputSink.decorate(Files.newByteChannel(path, writeOps));
64+
}
65+
66+
@Override
67+
public void close() throws IOException {
68+
Files.delete(path);
69+
onCloseCallback.run();
70+
}
71+
72+
@Override
73+
public String toString() {
74+
return MoreObjects.toStringHelper(this)
75+
.add("path", path)
76+
.add("throughputSink", throughputSink)
77+
.add("onCloseCallback", onCloseCallback)
78+
.toString();
79+
}
80+
81+
Unsafe unsafe() {
82+
return new Unsafe();
83+
}
84+
85+
final class Unsafe {
86+
public Path touch() throws UnsafeIOException {
87+
try {
88+
return RecoveryFile.this.touch();
89+
} catch (IOException e) {
90+
throw new UnsafeIOException(e);
91+
}
92+
}
93+
}
94+
95+
static final class UnsafeIOException extends RuntimeException {
96+
private UnsafeIOException(IOException cause) {
97+
super(cause);
98+
}
99+
}
100+
}

0 commit comments

Comments
 (0)