diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java index 408cdd3f3672..d242061226ef 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java @@ -51,9 +51,9 @@ import org.apache.paimon.table.source.TableRead; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; -import org.apache.paimon.types.TimestampType; import org.apache.paimon.utils.IteratorRecordReader; import org.apache.paimon.utils.ProjectedRow; import org.apache.paimon.utils.SerializationUtils; @@ -64,9 +64,6 @@ import javax.annotation.Nullable; import java.io.IOException; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -96,7 +93,7 @@ public class SnapshotsTable implements ReadonlyTable { new DataField(3, "commit_identifier", new BigIntType(false)), new DataField( 4, "commit_kind", SerializationUtils.newStringType(false)), - new DataField(5, "commit_time", new TimestampType(false, 3)), + new DataField(5, "commit_time", new LocalZonedTimestampType(false, 3)), new DataField( 6, "base_manifest_list", @@ -359,10 +356,7 @@ private InternalRow toRow(Snapshot snapshot) { BinaryString.fromString(snapshot.commitUser()), snapshot.commitIdentifier(), BinaryString.fromString(snapshot.commitKind().toString()), - Timestamp.fromLocalDateTime( - LocalDateTime.ofInstant( - Instant.ofEpochMilli(snapshot.timeMillis()), - ZoneId.systemDefault())), + Timestamp.fromEpochMillis(snapshot.timeMillis()), BinaryString.fromString(snapshot.baseManifestList()), BinaryString.fromString(snapshot.deltaManifestList()), BinaryString.fromString(snapshot.changelogManifestList()), diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java index f5a178998877..458f42b21d6a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java @@ -46,9 +46,9 @@ import org.apache.paimon.tag.Tag; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.RowType; import org.apache.paimon.types.TimestampType; -import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.IteratorRecordReader; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.ProjectedRow; @@ -87,7 +87,7 @@ public class TagsTable implements ReadonlyTable { new DataField(0, TAG_NAME, SerializationUtils.newStringType(false)), new DataField(1, "snapshot_id", new BigIntType(false)), new DataField(2, "schema_id", new BigIntType(false)), - new DataField(3, "commit_time", new TimestampType(false, 3)), + new DataField(3, "commit_time", new LocalZonedTimestampType(false, 3)), new DataField(4, "record_count", new BigIntType(true)), new DataField(5, "create_time", new TimestampType(true, 3)), new DataField( @@ -296,7 +296,7 @@ private InternalRow toRow(Map.Entry snapshot) { BinaryString.fromString(snapshot.getKey()), tag.id(), tag.schemaId(), - Timestamp.fromLocalDateTime(DateTimeUtils.toLocalDateTime(tag.timeMillis())), + Timestamp.fromEpochMillis(tag.timeMillis()), tag.totalRecordCount(), Optional.ofNullable(tag.getTagCreateTime()) .map(Timestamp::fromLocalDateTime) diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java index 81633ee7827c..2968c8d9d7c0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java @@ -45,6 +45,7 @@ import org.apache.paimon.table.TableTestBase; import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.utils.SnapshotManager; @@ -52,9 +53,6 @@ import org.junit.jupiter.api.Test; import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -106,6 +104,12 @@ public void before() throws Exception { @Test public void testReadSnapshotsFromLatest() throws Exception { + assertThat(snapshotsTable.rowType().getField(5)) + .isEqualTo( + new DataField( + 5, + "commit_time", + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).notNull())); List expectedRow = getExpectedResult(new long[] {1, 2}); List result = read(snapshotsTable); assertThat(result).containsExactlyInAnyOrderElementsOf(expectedRow); @@ -278,10 +282,7 @@ private List getExpectedResult(long[] snapshotIds) { BinaryString.fromString(snapshot.commitUser()), snapshot.commitIdentifier(), BinaryString.fromString(snapshot.commitKind().toString()), - Timestamp.fromLocalDateTime( - LocalDateTime.ofInstant( - Instant.ofEpochMilli(snapshot.timeMillis()), - ZoneId.systemDefault())), + Timestamp.fromEpochMillis(snapshot.timeMillis()), BinaryString.fromString(snapshot.baseManifestList()), BinaryString.fromString(snapshot.deltaManifestList()), BinaryString.fromString(snapshot.changelogManifestList()), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/TagsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/TagsTableTest.java index 5cb2c47b0bba..3f79168f1932 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/TagsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/TagsTableTest.java @@ -35,8 +35,8 @@ import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.tag.Tag; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; -import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.TagManager; @@ -95,6 +95,12 @@ void before() throws Exception { @Test void testTagsTable() throws Exception { + assertThat(tagsTable.rowType().getField(3)) + .isEqualTo( + new DataField( + 3, + "commit_time", + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).notNull())); List expectRow = getExpectedResult(); List result = read(tagsTable); assertThat(result).containsExactlyElementsOf(expectRow); @@ -208,8 +214,7 @@ private List getExpectedResult() { BinaryString.fromString(tagName), tag.id(), tag.schemaId(), - Timestamp.fromLocalDateTime( - DateTimeUtils.toLocalDateTime(tag.timeMillis())), + Timestamp.fromEpochMillis(tag.timeMillis()), tag.totalRecordCount(), tag.getTagCreateTime() == null ? null diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonSystemTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonSystemTableTest.scala index 48f8c3b2697d..b15f03648e68 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonSystemTableTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonSystemTableTest.scala @@ -137,4 +137,27 @@ class PaimonSystemTableTest extends PaimonSparkTestBase { sql("CALL paimon.sys.expire_snapshots(table => 'test.T', retain_max => 1, retain_min => 1)") checkAnswer(sql("SELECT snapshot_id FROM `T$snapshots` WHERE snapshot_id < 2"), Nil) } + + test("system table: snapshot and tag commit times use the snapshot instant") { + assume(gteqSpark3_4) + sql("CREATE TABLE T (a INT)") + sql("INSERT INTO T VALUES (1)") + sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'first', snapshot => 1)") + val millis = loadTable("T").snapshotManager().snapshot(1).timeMillis() + + withTimeZone("Asia/Shanghai") { + withSparkSQLConf( + "spark.sql.session.timeZone" -> "UTC", + "spark.paimon.legacy-timestamp-mapping.enabled" -> "false") { + checkAnswer( + sql(s""" + |SELECT unix_millis(s.commit_time), unix_millis(t.commit_time) + |FROM `T$$snapshots` s JOIN `T$$tags` t ON s.snapshot_id = t.snapshot_id + |WHERE t.tag_name = 'first' AND s.commit_time < timestamp_millis(${millis + 1}) + |""".stripMargin), + Seq(Row(millis, millis)) + ) + } + } + } }