Skip to content

docs: samples and tests for backup Admin APIs and overall spanner Admin APIs. #2882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 44 commits into from
Feb 23, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
edc5bbf
fix: prevent illegal negative timeout values into thread sleep() meth…
arpan14 Feb 6, 2023
49a85df
Merge pull request #1 from arpan14/retryerror
arpan14 Feb 8, 2023
4cd497b
Fixing lint issues.
arpan14 Feb 8, 2023
4a6aa8e
Merge branch 'googleapis:main' into main
arpan14 Mar 13, 2023
b2aa09d
Merge branch 'googleapis:main' into main
arpan14 Mar 15, 2023
8d6d71e
Merge branch 'googleapis:main' into main
arpan14 May 9, 2023
77e6e7d
Merge branch 'googleapis:main' into main
arpan14 Jul 17, 2023
e8b7fad
Merge branch 'googleapis:main' into main
arpan14 Jul 25, 2023
8aa84e1
Merge branch 'googleapis:main' into main
arpan14 Oct 10, 2023
57fd405
Merge branch 'googleapis:main' into main
arpan14 Oct 27, 2023
1253563
Merge branch 'googleapis:main' into main
arpan14 Nov 20, 2023
d4f6a60
Merge branch 'googleapis:main' into main
arpan14 Dec 15, 2023
3efaf7c
Merge branch 'googleapis:main' into main
arpan14 Dec 26, 2023
f41b39f
Merge branch 'googleapis:main' into main
arpan14 Jan 3, 2024
3775996
chore: copy samples and rewrite create backup.
arpan14 Jan 10, 2024
c9d60df
chore: fix code.
arpan14 Jan 10, 2024
8989e7c
chore: fix code.
arpan14 Jan 10, 2024
1fedc6e
Merge branch 'googleapis:main' into admin-backup-samples
arpan14 Feb 13, 2024
4e84671
chore: fix code.
arpan14 Jan 11, 2024
3c0d0f6
chore: fix error.
arpan14 Feb 13, 2024
8e51bde
fix: all compilation errors.
arpan14 Feb 15, 2024
fb9ad16
fix: more compilation issues.
arpan14 Feb 15, 2024
0182569
fix: all compile issues.
arpan14 Feb 15, 2024
aa8faca
chore: compile and test PgSpannerSample.
arpan14 Feb 16, 2024
b806ef1
fix: all issues with PgSpannerSample.
arpan14 Feb 16, 2024
7d5943d
chore: compile and fix SpannerSample/SpannerSampleIT.
arpan14 Feb 17, 2024
db5cbca
chore: fix kms configs and ITs.
arpan14 Feb 19, 2024
9240fbc
chore: fix cancel backup tests.
arpan14 Feb 19, 2024
e752d32
Update samples/snippets/src/main/java/com/example/spanner/admin/gener…
arpan14 Feb 19, 2024
ca59fb6
chore: add log statements in tests.
arpan14 Feb 20, 2024
fcf4457
chore: fix lint errors and fix comments.
arpan14 Feb 20, 2024
89834e0
chore: fix lint errors.
arpan14 Feb 20, 2024
478c343
chore: fix lint errors.
arpan14 Feb 20, 2024
d719f7f
chore: fix backup samples for restore use-case.
arpan14 Feb 22, 2024
d324ed6
fix: fix restore/list backup tests.
arpan14 Feb 22, 2024
837034a
chore: fix PgSpannerSample sample and test.
arpan14 Feb 22, 2024
f7623ed
chore: fix delete backup test.
arpan14 Feb 23, 2024
3f90c1a
chore: fix lint errors.
arpan14 Feb 23, 2024
bb7a6cc
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 23, 2024
6f36d93
Merge branch 'main' into admin-backup-samples
arpan14 Feb 23, 2024
01058bc
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Feb 23, 2024
0dcce23
chore: add IT for CopyBackupSample.
arpan14 Feb 23, 2024
fa88a45
chore: fix lint errors.
arpan14 Feb 23, 2024
e3630bc
chore: refactor test for delete backup.
arpan14 Feb 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
chore: fix PgSpannerSample sample and test.
  • Loading branch information
arpan14 committed Feb 22, 2024
commit 837034aeac024bf840997dd375d7cd6f88ae2325
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,6 @@
import com.google.cloud.Timestamp;
import com.google.cloud.spanner.DatabaseClient;
import com.google.cloud.spanner.DatabaseId;
import com.google.cloud.spanner.Instance;
import com.google.cloud.spanner.InstanceAdminClient;
import com.google.cloud.spanner.InstanceId;
import com.google.cloud.spanner.Key;
import com.google.cloud.spanner.KeyRange;
import com.google.cloud.spanner.KeySet;
Expand All @@ -42,14 +39,20 @@
import com.google.cloud.spanner.TimestampBound;
import com.google.cloud.spanner.Value;
import com.google.cloud.spanner.admin.database.v1.DatabaseAdminClient;
import com.google.cloud.spanner.admin.database.v1.DatabaseAdminClient.ListBackupOperationsPagedResponse;
import com.google.cloud.spanner.admin.database.v1.DatabaseAdminClient.ListDatabaseOperationsPagedResponse;
import com.google.common.io.BaseEncoding;
import com.google.longrunning.Operation;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.spanner.admin.database.v1.BackupName;
import com.google.spanner.admin.database.v1.CopyBackupMetadata;
import com.google.spanner.admin.database.v1.CreateBackupMetadata;
import com.google.spanner.admin.database.v1.CreateDatabaseRequest;
import com.google.spanner.admin.database.v1.Database;
import com.google.spanner.admin.database.v1.DatabaseDialect;
import com.google.spanner.admin.database.v1.DatabaseName;
import com.google.spanner.admin.database.v1.ListBackupOperationsRequest;
import com.google.spanner.admin.database.v1.ListDatabaseOperationsRequest;
import com.google.spanner.admin.database.v1.OptimizeRestoredDatabaseMetadata;
import com.google.spanner.admin.instance.v1.InstanceName;
import com.google.spanner.v1.ExecuteSqlRequest;
Expand Down Expand Up @@ -1240,22 +1243,25 @@ static void queryWithQueryOptions(DatabaseClient dbClient) {
// [END spanner_postgresql_query_with_query_options]

// [START spanner_postgresql_list_backup_operations]
static void listBackupOperations(InstanceAdminClient instanceAdminClient, DatabaseId databaseId) {
Instance instance = instanceAdminClient.getInstance(databaseId.getInstanceId().getInstance());
// Get create backup operations for the sample database.
Timestamp last24Hours = Timestamp.ofTimeSecondsAndNanos(TimeUnit.SECONDS.convert(
TimeUnit.HOURS.convert(Timestamp.now().getSeconds(), TimeUnit.SECONDS) - 24,
TimeUnit.HOURS), 0);
static void listBackupOperations(
DatabaseAdminClient databaseAdminClient,
String projectId, String instanceId,
String databaseId, String backupId) {
com.google.spanner.admin.database.v1.InstanceName instanceName = com.google.spanner.admin.database.v1.InstanceName.of(projectId, instanceId);
// Get 'CreateBackup' operations for the sample database.
String filter =
String.format(
"(metadata.database:%s) AND "
+ "(metadata.@type:type.googleapis.com/"
+ "google.spanner.admin.database.v1.CreateBackupMetadata) AND "
+ "(metadata.progress.start_time > \"%s\")",
databaseId.getName(), last24Hours);
Page<Operation> operations = instance
.listBackupOperations(Options.filter(filter));
for (Operation op : operations.iterateAll()) {
"(metadata.@type:type.googleapis.com/"
+ "google.spanner.admin.database.v1.CreateBackupMetadata) "
+ "AND (metadata.database:%s)",
DatabaseName.of(projectId, instanceId, databaseId).toString());
ListBackupOperationsRequest listBackupOperationsRequest =
ListBackupOperationsRequest.newBuilder()
.setParent(instanceName.toString()).setFilter(filter).build();
ListBackupOperationsPagedResponse createBackupOperations
= databaseAdminClient.listBackupOperations(listBackupOperationsRequest);
System.out.println("Create Backup Operations:");
for (Operation op : createBackupOperations.iterateAll()) {
try {
CreateBackupMetadata metadata = op.getMetadata().unpack(CreateBackupMetadata.class);
System.out.println(
Expand All @@ -1269,23 +1275,53 @@ static void listBackupOperations(InstanceAdminClient instanceAdminClient, Databa
System.err.println(e.getMessage());
}
}
// Get copy backup operations for the sample database.
filter = String.format(
"(metadata.@type:type.googleapis.com/"
+ "google.spanner.admin.database.v1.CopyBackupMetadata) "
+ "AND (metadata.source_backup:%s)",
BackupName.of(projectId, instanceId, backupId).toString());
listBackupOperationsRequest =
ListBackupOperationsRequest.newBuilder()
.setParent(instanceName.toString()).setFilter(filter).build();
ListBackupOperationsPagedResponse copyBackupOperations =
databaseAdminClient.listBackupOperations(listBackupOperationsRequest);
System.out.println("Copy Backup Operations:");
for (Operation op : copyBackupOperations.iterateAll()) {
try {
CopyBackupMetadata copyBackupMetadata =
op.getMetadata().unpack(CopyBackupMetadata.class);
System.out.println(
String.format(
"Copy Backup %s on backup %s pending: %d%% complete",
copyBackupMetadata.getName(),
copyBackupMetadata.getSourceBackup(),
copyBackupMetadata.getProgress().getProgressPercent()));
} catch (InvalidProtocolBufferException e) {
// The returned operation does not contain CopyBackupMetadata.
System.err.println(e.getMessage());
}
}
}
// [END spanner_postgresql_list_backup_operations]

// [START spanner_postgresql_list_database_operations]
static void listDatabaseOperations(
InstanceAdminClient instanceAdminClient,
DatabaseAdminClient dbAdminClient,
InstanceId instanceId) {
Instance instance = instanceAdminClient.getInstance(instanceId.getInstance());
DatabaseAdminClient dbAdminClient, String projectId, String instanceId) {
// Get optimize restored database operations.
Timestamp last24Hours = Timestamp.ofTimeSecondsAndNanos(TimeUnit.SECONDS.convert(
TimeUnit.HOURS.convert(Timestamp.now().getSeconds(), TimeUnit.SECONDS) - 24,
com.google.cloud.Timestamp last24Hours = com.google.cloud.Timestamp.ofTimeSecondsAndNanos(TimeUnit.SECONDS.convert(
TimeUnit.HOURS.convert(com.google.cloud.Timestamp.now().getSeconds(), TimeUnit.SECONDS) - 24,
TimeUnit.HOURS), 0);
String filter = String.format("(metadata.@type:type.googleapis.com/"
+ "google.spanner.admin.database.v1.OptimizeRestoredDatabaseMetadata) AND "
+ "(metadata.progress.start_time > \"%s\")", last24Hours);
for (Operation op : instance.listDatabaseOperations(Options.filter(filter)).iterateAll()) {
ListDatabaseOperationsRequest listDatabaseOperationsRequest =
ListDatabaseOperationsRequest.newBuilder()
.setParent(com.google.spanner.admin.instance.v1.InstanceName.of(
projectId, instanceId).toString()).setFilter(filter).build();
ListDatabaseOperationsPagedResponse pagedResponse
= dbAdminClient.listDatabaseOperations(listDatabaseOperationsRequest);
for (Operation op : pagedResponse.iterateAll()) {
try {
OptimizeRestoredDatabaseMetadata metadata =
op.getMetadata().unpack(OptimizeRestoredDatabaseMetadata.class);
Expand All @@ -1304,9 +1340,9 @@ static void listDatabaseOperations(
static void run(
DatabaseClient dbClient,
DatabaseAdminClient dbAdminClient,
InstanceAdminClient instanceAdminClient,
String command,
DatabaseId database) {
DatabaseId database,
String backupId) {
DatabaseName databaseName = DatabaseName.of(database.getInstanceId().getProject(),
database.getInstanceId().getInstance(), database.getDatabase());
switch (command) {
Expand Down Expand Up @@ -1444,10 +1480,12 @@ static void run(
queryWithQueryOptions(dbClient);
break;
case "listbackupoperations":
listBackupOperations(instanceAdminClient, database);
listBackupOperations(dbAdminClient, database.getInstanceId().getProject(),
database.getInstanceId().getInstance(), database.getDatabase(), backupId);
break;
case "listdatabaseoperations":
listDatabaseOperations(instanceAdminClient, dbAdminClient, database.getInstanceId());
listDatabaseOperations(dbAdminClient, database.getInstanceId().getProject(),
database.getInstanceId().getInstance());
break;
default:
printUsageAndExit();
Expand Down Expand Up @@ -1525,14 +1563,19 @@ public static void main(String[] args) throws Exception {
+ clientProject);
printUsageAndExit();
}
// Generate a backup id for the sample database.
String backupId = null;
if (args.length == 4) {
backupId = args[3];
}

// [START spanner_init_client]
DatabaseClient dbClient = spanner.getDatabaseClient(db);
dbAdminClient = DatabaseAdminClient.create();
InstanceAdminClient instanceAdminClient = spanner.getInstanceAdminClient();
// [END spanner_init_client]

// Use client here...
run(dbClient, dbAdminClient, instanceAdminClient, command, db);
run(dbClient, dbAdminClient, command, db, backupId);
// [START spanner_init_client]
} finally {
if (dbAdminClient != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,14 @@
import com.google.cloud.spanner.Spanner;
import com.google.cloud.spanner.SpannerOptions;
import com.google.cloud.spanner.admin.database.v1.DatabaseAdminClient;
import com.google.spanner.admin.database.v1.BackupName;
import com.google.spanner.admin.database.v1.Database;
import com.google.spanner.admin.database.v1.DatabaseName;
import com.google.spanner.admin.database.v1.InstanceName;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
Expand All @@ -49,10 +46,6 @@ public class PgSpannerSampleIT extends SampleTestBaseV2 {
// The instance needs to exist for tests to pass.
private static final String instanceId = System.getProperty("spanner.test.instance");
private static final String baseDbId = System.getProperty("spanner.sample.database");
private static final String databaseId = formatForTest(baseDbId);
private static final String encryptedDatabaseId = formatForTest(baseDbId);
private static final String encryptedBackupId = formatForTest(baseDbId);
private static final String encryptedRestoreId = formatForTest(baseDbId);
static Spanner spanner;
static DatabaseId dbId;
static DatabaseAdminClient dbClient;
Expand All @@ -63,7 +56,7 @@ public static void setUp() throws IOException {
SpannerOptions.newBuilder().setAutoThrottleAdministrativeRequests().build();
spanner = options.getService();
dbClient = DatabaseAdminClient.create();
dbId = DatabaseId.of(options.getProjectId(), instanceId, databaseId);
dbId = DatabaseId.of(options.getProjectId(), instanceId, idGenerator.generateDatabaseId());
// Delete stale test databases that have been created earlier by this test, but not deleted.
deleteStaleTestDatabases();
}
Expand Down Expand Up @@ -91,19 +84,6 @@ static void deleteStaleTestDatabases() {
}
}

@AfterClass
public static void tearDown() {
dbClient.dropDatabase(
DatabaseName.of(projectId, dbId.getInstanceId().getInstance(), dbId.getDatabase()));
dbClient.dropDatabase(
DatabaseName.of(projectId, dbId.getInstanceId().getInstance(),
SpannerSample.createRestoredSampleDbId(dbId)));
dbClient.dropDatabase(DatabaseName.of(projectId, instanceId, encryptedDatabaseId));
dbClient.dropDatabase(DatabaseName.of(projectId, instanceId, encryptedRestoreId));
dbClient.deleteBackup(BackupName.of(projectId, instanceId, encryptedBackupId));
spanner.close();
}

private static String toComparableId(String baseId, String existingId) {
String zeroUuid = "00000000-0000-0000-0000-0000-00000000";
int shouldBeLength = (baseId + "-" + zeroUuid).length();
Expand All @@ -117,25 +97,21 @@ private static Pattern getTestDbIdPattern(String baseDbId) {
Pattern.CASE_INSENSITIVE);
}

static String formatForTest(String name) {
return name + "-" + UUID.randomUUID().toString().substring(0, DBID_LENGTH);
}

private String runSample(String command) throws Exception {
final PrintStream stdOut = System.out;
final ByteArrayOutputStream bout = new ByteArrayOutputStream();
final PrintStream out = new PrintStream(bout);
System.setOut(out);
System.out.println(instanceId + ":" + databaseId);
PgSpannerSample.main(new String[]{command, instanceId, databaseId});
System.out.println(instanceId + ":" + dbId.getDatabase());
PgSpannerSample.main(new String[]{command, instanceId, dbId.getDatabase()});
System.setOut(stdOut);
return bout.toString();
}

@Test
public void testSample() throws Exception {
assertThat(instanceId).isNotNull();
assertThat(databaseId).isNotNull();
assertThat(dbId.getDatabase()).isNotNull();

System.out.println("Create Database ...");
String out = runSample("createpgdatabase");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ public class SpannerSampleIT extends SampleTestBaseV2 {
Preconditions.checkNotNull(System.getProperty("spanner.test.key.ring"));
private static final String keyName =
Preconditions.checkNotNull(System.getProperty("spanner.test.key.name"));
private static final String encryptedDatabaseId = formatForTest(baseDbId);
private static final String encryptedBackupId = formatForTest(baseDbId);
private static final long STALE_INSTANCE_THRESHOLD_SECS =
TimeUnit.SECONDS.convert(24L, TimeUnit.HOURS);
Expand Down Expand Up @@ -171,7 +170,6 @@ static void deleteStaleTestDatabases() throws IOException {

@AfterClass
public static void tearDown() {
databaseAdminClient.dropDatabase(DatabaseName.of(projectId, instanceId, encryptedDatabaseId));
databaseAdminClient.deleteBackup(BackupName.of(projectId, instanceId, encryptedBackupId));
spanner.close();
}
Expand Down Expand Up @@ -409,7 +407,7 @@ public void testBackupSamples_withoutEncryption() {
}

System.out.println("List Backup ...");
out = runSample("listbackups", databaseId);
out = runSample("listbackups", databaseId, backupId);
assertThat(out).contains("All backups:");
assertThat(out).contains(
String.format("All backups with backup name containing \"%s\":", backupId));
Expand All @@ -422,8 +420,8 @@ public void testBackupSamples_withoutEncryption() {
assertThat(out).containsMatch(
Pattern.compile("All databases created after (.+) and that are ready:"));
assertThat(out).contains("All backups, listed using pagination:");
// All the above tests should include the created backup exactly once, i.e. exactly 7 times.
assertThat(countOccurrences(out, backupName.toString())).isEqualTo(7);
// All the above tests should include the created backup exactly once, i.e. exactly 6 times.
assertThat(countOccurrences(out, backupName.toString())).isEqualTo(6);

// Try the restore operation in a retry loop, as there is a limit on the number of restore
// operations that is allowed to execute simultaneously, and we should retry if we hit this
Expand Down