Skip to content

Commit 257edfc

Browse files
author
Praful Makani
authored
docs(samples): add query external bigtable using permanent table (#764)
* docs(samples): add query external bigtable using permanent table * docs(samples): address feedback * docs(samples): address comments
1 parent a5573d1 commit 257edfc

File tree

2 files changed

+312
-0
lines changed

2 files changed

+312
-0
lines changed

‎samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtablePerm.java

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquery;
18+
19+
// [START bigquery_query_external_bigtable_perm]
20+
import com.google.cloud.bigquery.BigQuery;
21+
import com.google.cloud.bigquery.BigQueryException;
22+
import com.google.cloud.bigquery.BigQueryOptions;
23+
import com.google.cloud.bigquery.BigtableColumn;
24+
import com.google.cloud.bigquery.BigtableColumnFamily;
25+
import com.google.cloud.bigquery.BigtableOptions;
26+
import com.google.cloud.bigquery.ExternalTableDefinition;
27+
import com.google.cloud.bigquery.QueryJobConfiguration;
28+
import com.google.cloud.bigquery.TableId;
29+
import com.google.cloud.bigquery.TableInfo;
30+
import com.google.cloud.bigquery.TableResult;
31+
import com.google.common.collect.ImmutableList;
32+
import org.apache.commons.codec.binary.Base64;
33+
34+
// Sample to queries an external bigtable data source using a permanent table
35+
public class QueryExternalBigtablePerm {
36+
37+
public static void main(String[] args) {
38+
// TODO(developer): Replace these variables before running the sample.
39+
String projectId = "MY_PROJECT_ID";
40+
String bigtableInstanceId = "MY_INSTANCE_ID";
41+
String bigtableTableName = "MY_BIGTABLE_NAME";
42+
String bigqueryDatasetName = "MY_DATASET_NAME";
43+
String bigqueryTableName = "MY_TABLE_NAME";
44+
String sourceUri =
45+
String.format(
46+
"https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
47+
projectId, bigtableInstanceId, bigtableTableName);
48+
String query = String.format("SELECT * FROM %s ", bigqueryTableName);
49+
queryExternalBigtablePerm(bigqueryDatasetName, bigqueryTableName, sourceUri, query);
50+
}
51+
52+
public static void queryExternalBigtablePerm(
53+
String datasetName, String tableName, String sourceUri, String query) {
54+
try {
55+
// Initialize client that will be used to send requests. This client only needs to be created
56+
// once, and can be reused for multiple requests.
57+
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
58+
59+
BigtableColumnFamily.Builder statsSummary = BigtableColumnFamily.newBuilder();
60+
61+
// Configuring Columns
62+
BigtableColumn connectedCell =
63+
BigtableColumn.newBuilder()
64+
.setQualifierEncoded(Base64.encodeBase64String("connected_cell".getBytes()))
65+
.setFieldName("connected_cell")
66+
.setType("STRING")
67+
.setEncoding("TEXT")
68+
.build();
69+
BigtableColumn connectedWifi =
70+
BigtableColumn.newBuilder()
71+
.setQualifierEncoded(Base64.encodeBase64String("connected_wifi".getBytes()))
72+
.setFieldName("connected_wifi")
73+
.setType("STRING")
74+
.setEncoding("TEXT")
75+
.build();
76+
BigtableColumn osBuild =
77+
BigtableColumn.newBuilder()
78+
.setQualifierEncoded(Base64.encodeBase64String("os_build".getBytes()))
79+
.setFieldName("os_build")
80+
.setType("STRING")
81+
.setEncoding("TEXT")
82+
.build();
83+
84+
// Configuring column family and columns
85+
statsSummary
86+
.setColumns(ImmutableList.of(connectedCell, connectedWifi, osBuild))
87+
.setFamilyID("stats_summary")
88+
.setOnlyReadLatest(true)
89+
.setEncoding("TEXT")
90+
.setType("STRING")
91+
.build();
92+
93+
// Configuring BigtableOptions is optional.
94+
BigtableOptions options =
95+
BigtableOptions.newBuilder()
96+
.setIgnoreUnspecifiedColumnFamilies(true)
97+
.setReadRowkeyAsString(true)
98+
.setColumnFamilies(ImmutableList.of(statsSummary.build()))
99+
.build();
100+
101+
TableId tableId = TableId.of(datasetName, tableName);
102+
// Create a permanent table linked to the Bigtable table
103+
ExternalTableDefinition externalTable =
104+
ExternalTableDefinition.newBuilder(sourceUri, options).build();
105+
bigquery.create(TableInfo.of(tableId, externalTable));
106+
107+
// Example query
108+
TableResult results = bigquery.query(QueryJobConfiguration.of(query));
109+
110+
results
111+
.iterateAll()
112+
.forEach(row -> row.forEach(val -> System.out.printf("%s,", val.toString())));
113+
114+
System.out.println("Query on external permanent table performed successfully.");
115+
} catch (BigQueryException | InterruptedException e) {
116+
System.out.println("Query not performed \n" + e.toString());
117+
}
118+
}
119+
}
120+
// [END bigquery_query_external_bigtable_perm]

‎samples/snippets/src/test/java/com/example/bigquery/QueryExternalBigtablePermIT.java

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquery;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
import static junit.framework.TestCase.assertNotNull;
21+
22+
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
23+
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
24+
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
25+
import com.google.cloud.bigtable.data.v2.models.BulkMutation;
26+
import com.google.cloud.bigtable.data.v2.models.Mutation;
27+
import com.google.protobuf.ByteString;
28+
import java.io.ByteArrayOutputStream;
29+
import java.io.IOException;
30+
import java.io.PrintStream;
31+
import java.util.UUID;
32+
import java.util.logging.Level;
33+
import java.util.logging.Logger;
34+
import org.junit.After;
35+
import org.junit.Before;
36+
import org.junit.BeforeClass;
37+
import org.junit.Test;
38+
39+
public class QueryExternalBigtablePermIT {
40+
41+
private final Logger log = Logger.getLogger(this.getClass().getName());
42+
private static final String ID = UUID.randomUUID().toString().substring(0, 8);
43+
private static final String TABLE_ID = "bigquery-samples-test" + ID;
44+
private static final String COLUMN_FAMILY_NAME = "stats_summary";
45+
private static final long TIMESTAMP = System.currentTimeMillis() * 1000;
46+
private static final String CONNECTED_CELL = "connected_cell";
47+
private static final String CONNECTED_WIFI = "connected_wifi";
48+
private static final String OS_BUILD = "os_build";
49+
private String tableName;
50+
private ByteArrayOutputStream bout;
51+
private PrintStream out;
52+
private PrintStream originalPrintStream;
53+
54+
private static final String INSTANCE = requireEnvVar("BIGTABLE_TESTING_INSTANCE");
55+
private static final String PROJECT = requireEnvVar("SAMPLES_TESTING_PROJECT");
56+
private static final String BIGQUERY_DATASET_NAME = requireEnvVar("BIGQUERY_DATASET_NAME");
57+
58+
private static String requireEnvVar(String varName) {
59+
String value = System.getenv(varName);
60+
assertNotNull(
61+
"Environment variable " + varName + " is required to perform these tests.",
62+
System.getenv(varName));
63+
return value;
64+
}
65+
66+
@BeforeClass
67+
public static void checkRequirements() {
68+
requireEnvVar("GOOGLE_CLOUD_PROJECT");
69+
requireEnvVar("BIGTABLE_TESTING_INSTANCE");
70+
requireEnvVar("BIGQUERY_DATASET_NAME");
71+
}
72+
73+
@Before
74+
public void setUp() throws IOException {
75+
// Create a test table
76+
tableName = "EXTERNAL_TABLE_FROM_BIGTABLE_TEST_" + ID;
77+
bout = new ByteArrayOutputStream();
78+
out = new PrintStream(bout);
79+
originalPrintStream = System.out;
80+
System.setOut(out);
81+
82+
// create a temporary bigtable table.
83+
try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
84+
CreateTableRequest createTableRequest =
85+
CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_NAME);
86+
client.createTable(createTableRequest);
87+
}
88+
// inserting temporary rows.
89+
try (BigtableDataClient client = BigtableDataClient.create(PROJECT, INSTANCE)) {
90+
BulkMutation bulkMutation =
91+
BulkMutation.create(TABLE_ID)
92+
.add(
93+
"phone#4c410523#20190501",
94+
Mutation.create()
95+
.setCell(
96+
COLUMN_FAMILY_NAME,
97+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
98+
TIMESTAMP,
99+
1)
100+
.setCell(
101+
COLUMN_FAMILY_NAME,
102+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
103+
TIMESTAMP,
104+
1)
105+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.003"))
106+
.add(
107+
"phone#4c410523#20190502",
108+
Mutation.create()
109+
.setCell(
110+
COLUMN_FAMILY_NAME,
111+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
112+
TIMESTAMP,
113+
1)
114+
.setCell(
115+
COLUMN_FAMILY_NAME,
116+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
117+
TIMESTAMP,
118+
1)
119+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.004"))
120+
.add(
121+
"phone#4c410523#20190505",
122+
Mutation.create()
123+
.setCell(
124+
COLUMN_FAMILY_NAME,
125+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
126+
TIMESTAMP,
127+
0)
128+
.setCell(
129+
COLUMN_FAMILY_NAME,
130+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
131+
TIMESTAMP,
132+
1)
133+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"))
134+
.add(
135+
"phone#5c10102#20190501",
136+
Mutation.create()
137+
.setCell(
138+
COLUMN_FAMILY_NAME,
139+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
140+
TIMESTAMP,
141+
1)
142+
.setCell(
143+
COLUMN_FAMILY_NAME,
144+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
145+
TIMESTAMP,
146+
1)
147+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190401.002"))
148+
.add(
149+
"phone#5c10102#20190502",
150+
Mutation.create()
151+
.setCell(
152+
COLUMN_FAMILY_NAME,
153+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
154+
TIMESTAMP,
155+
1)
156+
.setCell(
157+
COLUMN_FAMILY_NAME,
158+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
159+
TIMESTAMP,
160+
0)
161+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"));
162+
163+
client.bulkMutateRows(bulkMutation);
164+
}
165+
}
166+
167+
@After
168+
public void tearDown() throws IOException {
169+
// Clean up
170+
try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
171+
client.deleteTable(TABLE_ID);
172+
}
173+
DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName);
174+
// restores print statements in the original method
175+
System.out.flush();
176+
System.setOut(originalPrintStream);
177+
log.log(Level.INFO, bout.toString());
178+
}
179+
180+
@Test
181+
public void testQueryExternalBigtablePerm() {
182+
String query = String.format("SELECT * FROM %s.%s ", BIGQUERY_DATASET_NAME, tableName);
183+
String sourceUri =
184+
String.format(
185+
"https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
186+
PROJECT, INSTANCE, TABLE_ID);
187+
QueryExternalBigtablePerm.queryExternalBigtablePerm(
188+
BIGQUERY_DATASET_NAME, tableName, sourceUri, query);
189+
assertThat(bout.toString())
190+
.contains("Query on external permanent table performed successfully.");
191+
}
192+
}

0 commit comments

Comments
 (0)