Skip to content

feat(bigquery): Add support for custom timezones and timestamps #3859

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
whuffman36 merged 6 commits into from
Jun 30, 2025
Merged

feat(bigquery): Add support for custom timezones and timestamps #3859

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5a04206
feat(bigquery): Add support for custom timezones and timestamps
whuffman36 Jun 23, 2025
2593cfe
update ignored-diff
whuffman36 Jun 23, 2025
099b71e
fix data -> date typo
whuffman36 Jun 23, 2025
681af66
Add enums for SourceColumnMatch
whuffman36 Jun 24, 2025
9efd587
Change null markers test names
whuffman36 Jun 24, 2025
ae64196
change enums to StringEnumValue
whuffman36 Jun 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions google-cloud-bigquery/clirr-ignored-differences.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,76 @@
<!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
<differences>
<!-- TODO: REMOVE AFTER RELEASE -->
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition*</className>
<method>*TimeZone(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition*</className>
<method>*DateFormat(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition*</className>
<method>*DatetimeFormat(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition*</className>
<method>*TimeFormat(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition*</className>
<method>*TimestampFormat(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition*</className>
<method>*SourceColumnMatch(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition*</className>
<method>*NullMarkers(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/LoadJobConfiguration*</className>
<method>*TimeZone(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/LoadJobConfiguration*</className>
<method>*DateFormat(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/LoadJobConfiguration*</className>
<method>*DatetimeFormat(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/LoadJobConfiguration*</className>
<method>*TimeFormat(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/LoadJobConfiguration*</className>
<method>*TimestampFormat(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/LoadJobConfiguration*</className>
<method>*SourceColumnMatch(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/LoadJobConfiguration*</className>
<method>*NullMarkers(*)</method>
</difference>
<difference>
<differenceType>7004</differenceType>
<className>com/google/cloud/bigquery/spi/v2/BigQueryRpc</className>
Expand Down
186 changes: 186 additions & 0 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Strings.isNullOrEmpty;

import com.google.api.core.ApiFunction;
import com.google.api.services.bigquery.model.ExternalDataConfiguration;
import com.google.api.services.bigquery.model.Table;
import com.google.auto.value.AutoValue;
import com.google.cloud.StringEnumType;
import com.google.cloud.StringEnumValue;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import java.util.List;
Expand Down Expand Up @@ -57,6 +60,46 @@ public ExternalDataConfiguration apply(ExternalTableDefinition tableInfo) {

private static final long serialVersionUID = -5951580238459622025L;

public static final class SourceColumnMatch extends StringEnumValue {
private static final long serialVersionUID = 818920627219751207L;
private static final ApiFunction<String, SourceColumnMatch> CONSTRUCTOR =
new ApiFunction<String, SourceColumnMatch>() {
@Override
public SourceColumnMatch apply(String constant) {
return new SourceColumnMatch(constant);
}
};

private static final StringEnumType<SourceColumnMatch> type =
new StringEnumType<SourceColumnMatch>(SourceColumnMatch.class, CONSTRUCTOR);

public static final SourceColumnMatch POSITION = type.createAndRegister("POSITION");

public static final SourceColumnMatch NAME = type.createAndRegister("NAME");

private SourceColumnMatch(String constant) {
super(constant);
}

/**
* Get the SourceColumnMatch for the given String constant, and throw an exception if the
* constant is not recognized.
*/
public static SourceColumnMatch valueOfStrict(String constant) {
return type.valueOfStrict(constant);
}

/** Get the SourceColumnMatch for the given String constant, and allow unrecognized values. */
public static SourceColumnMatch valueOf(String constant) {
return type.valueOf(constant);
}

/** Return the known values for SourceColumnMatch. */
public static SourceColumnMatch[] values() {
return type.values();
}
}

@AutoValue.Builder
public abstract static class Builder
extends TableDefinition.Builder<ExternalTableDefinition, Builder> {
Expand Down Expand Up @@ -215,6 +258,45 @@ public Builder setMaxStaleness(String maxStaleness) {

abstract Builder setMaxStalenessInner(String maxStaleness);

/**
* Time zone used when parsing timestamp values that do not have specific time zone information
* (e.g. 2024-04-20 12:34:56). The expected format is a IANA timezone string (e.g.
* America/Los_Angeles).
*/
public abstract Builder setTimeZone(String timeZone);

/** Format used to parse DATE values. Supports C-style and SQL-style values. */
public abstract Builder setDateFormat(String dateFormat);

/** Format used to parse DATETIME values. Supports C-style and SQL-style values. */
public abstract Builder setDatetimeFormat(String datetimeFormat);

/** Format used to parse TIME values. Supports C-style and SQL-style values. */
public abstract Builder setTimeFormat(String timeFormat);

/** Format used to parse TIMESTAMP values. Supports C-style and SQL-style values. */
public abstract Builder setTimestampFormat(String timestampFormat);

/**
* Controls the strategy used to match loaded columns to the schema. If not set, a sensible
* default is chosen based on how the schema is provided. If autodetect is used, then columns
* are matched by name. Otherwise, columns are matched by position. This is done to keep the
* behavior backward-compatible. Acceptable values are: POSITION - matches by position. This
* assumes that the columns are ordered the same way as the schema. NAME - matches by name. This
* reads the header row as column names and reorders columns to match the field names in the
* schema.
*/
public abstract Builder setSourceColumnMatch(SourceColumnMatch sourceColumnMatch);

/**
* A list of strings represented as SQL NULL value in a CSV file. null_marker and null_markers
* can't be set at the same time. If null_marker is set, null_markers has to be not set. If
* null_markers is set, null_marker has to be not set. If both null_marker and null_markers are
* set at the same time, a user error would be thrown. Any strings listed in null_markers,
* including empty string would be interpreted as SQL NULL. This applies to all column types.
*/
public abstract Builder setNullMarkers(List<String> nullMarkers);

/** Creates an {@code ExternalTableDefinition} object. */
@Override
public abstract ExternalTableDefinition build();
Expand Down Expand Up @@ -373,6 +455,37 @@ public HivePartitioningOptions getHivePartitioningOptions() {
return getHivePartitioningOptionsInner();
}

/**
* Returns the time zone used when parsing timestamp values that don't have specific time zone
* information.
*/
@Nullable
public abstract String getTimeZone();

/** Returns the format used to parse DATE values. */
@Nullable
public abstract String getDateFormat();

/** Returns the format used to parse DATETIME values. */
@Nullable
public abstract String getDatetimeFormat();

/** Returns the format used to parse TIME values. */
@Nullable
public abstract String getTimeFormat();

/** Returns the format used to parse TIMESTAMP values. */
@Nullable
public abstract String getTimestampFormat();

/** Returns the strategy used to match loaded columns to the schema, either POSITION or NAME. */
@Nullable
public abstract SourceColumnMatch getSourceColumnMatch();

/** Returns a list of strings represented as SQL NULL value in a CSV file. */
@Nullable
public abstract List<String> getNullMarkers();

@Nullable
abstract HivePartitioningOptions getHivePartitioningOptionsInner();

Expand Down Expand Up @@ -454,6 +567,29 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
if (getMetadataCacheMode() != null) {
externalConfigurationPb.setMetadataCacheMode(getMetadataCacheMode());
}
if (getTimeZone() != null) {
externalConfigurationPb.setTimeZone(getTimeZone());
}
if (getDateFormat() != null) {
externalConfigurationPb.setDateFormat(getDateFormat());
}
if (getDatetimeFormat() != null) {
externalConfigurationPb.setDatetimeFormat(getDatetimeFormat());
}
if (getTimeFormat() != null) {
externalConfigurationPb.setTimeFormat(getTimeFormat());
}
if (getTimestampFormat() != null) {
externalConfigurationPb.setTimestampFormat(getTimestampFormat());
}
if (getSourceColumnMatch() != null) {
externalConfigurationPb
.getCsvOptions()
.setSourceColumnMatch(getSourceColumnMatch().toString());
}
if (getNullMarkers() != null) {
externalConfigurationPb.getCsvOptions().setNullMarkers(getNullMarkers());
}

return externalConfigurationPb;
}
Expand Down Expand Up @@ -654,6 +790,31 @@ static ExternalTableDefinition fromPb(Table tablePb) {
if (tablePb.getMaxStaleness() != null) {
builder.setMaxStaleness(tablePb.getMaxStaleness());
}
if (externalDataConfiguration.getTimeZone() != null) {
builder.setTimeZone(externalDataConfiguration.getTimeZone());
}
if (externalDataConfiguration.getDateFormat() != null) {
builder.setDateFormat(externalDataConfiguration.getDateFormat());
}
if (externalDataConfiguration.getDatetimeFormat() != null) {
builder.setDatetimeFormat(externalDataConfiguration.getDatetimeFormat());
}
if (externalDataConfiguration.getTimeFormat() != null) {
builder.setTimeFormat(externalDataConfiguration.getTimeFormat());
}
if (externalDataConfiguration.getTimestampFormat() != null) {
builder.setTimestampFormat(externalDataConfiguration.getTimestampFormat());
}
if (externalDataConfiguration.getCsvOptions() != null) {
if (externalDataConfiguration.getCsvOptions().getSourceColumnMatch() != null) {
builder.setSourceColumnMatch(
SourceColumnMatch.valueOf(
externalDataConfiguration.getCsvOptions().getSourceColumnMatch()));
}
if (externalDataConfiguration.getCsvOptions().getNullMarkers() != null) {
builder.setNullMarkers(externalDataConfiguration.getCsvOptions().getNullMarkers());
}
}
}
return builder.build();
}
Expand Down Expand Up @@ -724,6 +885,31 @@ static ExternalTableDefinition fromExternalDataConfiguration(
if (externalDataConfiguration.getMetadataCacheMode() != null) {
builder.setMetadataCacheMode(externalDataConfiguration.getMetadataCacheMode());
}
if (externalDataConfiguration.getTimeZone() != null) {
builder.setTimeZone(externalDataConfiguration.getTimeZone());
}
if (externalDataConfiguration.getDateFormat() != null) {
builder.setDateFormat(externalDataConfiguration.getDateFormat());
}
if (externalDataConfiguration.getDatetimeFormat() != null) {
builder.setDatetimeFormat(externalDataConfiguration.getDatetimeFormat());
}
if (externalDataConfiguration.getTimeFormat() != null) {
builder.setTimeFormat(externalDataConfiguration.getTimeFormat());
}
if (externalDataConfiguration.getTimestampFormat() != null) {
builder.setTimestampFormat(externalDataConfiguration.getTimeFormat());
}
if (externalDataConfiguration.getCsvOptions() != null) {
if (externalDataConfiguration.getCsvOptions().getSourceColumnMatch() != null) {
builder.setSourceColumnMatch(
SourceColumnMatch.valueOf(
externalDataConfiguration.getCsvOptions().getSourceColumnMatch()));
}
if (externalDataConfiguration.getCsvOptions().getNullMarkers() != null) {
builder.setNullMarkers(externalDataConfiguration.getCsvOptions().getNullMarkers());
}
}

return builder.build();
}
Expand Down
Loading