Skip to content

Commit 00e5690

Browse files
committed
[SPARK-42516][SQL] Always capture the session time zone config while creating views
### What changes were proposed in this pull request? In the PR, I propose to capture the session time zone config (`spark.sql.session.timeZone`) as a view property, and use it while re-parsing/analysing the view. If the SQL config is not set while creating a view, use the default value of the config. ### Why are the changes needed? To improve user experience with Spark SQL. The current behaviour might confuse users because query results depends on whether or not the session time zone was set explicitly while creating a view. ### Does this PR introduce _any_ user-facing change? Yes. Before the changes, the current value of the session time zone is used in view analysis but this behaviour can be restored via another SQL config `spark.sql.legacy.useCurrentConfigsForView`. ### How was this patch tested? By running the new test via: ``` $ build/sbt "test:testOnly *.PersistedViewTestSuite" ``` Closes apache#40103 from MaxGekk/view-tz-conf. Authored-by: Max Gekk <[email protected]> Signed-off-by: Max Gekk <[email protected]>
1 parent a6098be commit 00e5690

File tree

2 files changed

+25
-1
lines changed

2 files changed

+25
-1
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -398,8 +398,15 @@ object ViewHelper extends SQLConfHelper with Logging {
398398
val modifiedConfs = conf.getAllConfs.filter { case (k, _) =>
399399
conf.isModifiable(k) && shouldCaptureConfig(k)
400400
}
401+
// Some configs have dynamic default values, such as SESSION_LOCAL_TIMEZONE whose
402+
// default value relies on the JVM system timezone. We need to always capture them to
403+
// to make sure we apply the same configs when reading the view.
404+
val alwaysCaptured = Seq(SQLConf.SESSION_LOCAL_TIMEZONE)
405+
.filter(c => !modifiedConfs.contains(c.key))
406+
.map(c => (c.key, conf.getConf(c)))
407+
401408
val props = new mutable.HashMap[String, String]
402-
for ((key, value) <- modifiedConfs) {
409+
for ((key, value) <- modifiedConfs ++ alwaysCaptured) {
403410
props.put(s"$VIEW_SQL_CONFIG_PREFIX$key", value)
404411
}
405412
props.toMap

sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
2424
import org.apache.spark.sql.catalyst.catalog.CatalogFunction
2525
import org.apache.spark.sql.catalyst.expressions.Expression
2626
import org.apache.spark.sql.catalyst.plans.logical.Repartition
27+
import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.withDefaultTimeZone
2728
import org.apache.spark.sql.connector.catalog._
2829
import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
2930
import org.apache.spark.sql.internal.SQLConf._
@@ -714,6 +715,22 @@ class PersistedViewTestSuite extends SQLViewTestSuite with SharedSparkSession {
714715
}
715716
}
716717

718+
test("capture the session time zone config while creating a view") {
719+
val viewName = "v1_capture_test"
720+
withView(viewName) {
721+
assert(get.sessionLocalTimeZone === "America/Los_Angeles")
722+
createView(viewName,
723+
"""select hour(ts) as H from (
724+
| select cast('2022-01-01T00:00:00.000 America/Los_Angeles' as timestamp) as ts
725+
|)""".stripMargin, Seq("H"))
726+
withDefaultTimeZone(java.time.ZoneId.of("UTC-09:00")) {
727+
withSQLConf(SESSION_LOCAL_TIMEZONE.key -> "UTC-10:00") {
728+
checkAnswer(sql(s"select H from $viewName"), Row(0))
729+
}
730+
}
731+
}
732+
}
733+
717734
def getShowCreateDDL(view: String, serde: Boolean = false): String = {
718735
val result = if (serde) {
719736
sql(s"SHOW CREATE TABLE $view AS SERDE")

0 commit comments

Comments
 (0)