From eeffff222c4c5ffd5bb75c88930531be759b4586 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 26 Oct 2023 15:20:02 +0200 Subject: [PATCH] Respect of BQ time format spec (#850) --- .../bigquery/TimestampConverter.scala | 114 ++++++++++++------ .../bigquery/TimestampConverterSuite.scala | 106 ++++++++++++++++ 2 files changed, 184 insertions(+), 36 deletions(-) create mode 100644 bigquery/src/test/scala/magnolify/bigquery/TimestampConverterSuite.scala diff --git a/bigquery/src/main/scala/magnolify/bigquery/TimestampConverter.scala b/bigquery/src/main/scala/magnolify/bigquery/TimestampConverter.scala index 3c71c751b..93998f144 100644 --- a/bigquery/src/main/scala/magnolify/bigquery/TimestampConverter.scala +++ b/bigquery/src/main/scala/magnolify/bigquery/TimestampConverter.scala @@ -22,58 +22,100 @@ import java.time.temporal.ChronoField // https://github.com/googleapis/java-bigquery/blob/master/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java private object TimestampConverter { + // TIME + // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#time_type + // [H]H:[M]M:[S]S[.DDDDDD|.F] + private val timeFormatter = new DateTimeFormatterBuilder() + .appendValue(ChronoField.HOUR_OF_DAY, 1, 2, SignStyle.NEVER) + .appendLiteral(':') + .appendValue(ChronoField.MINUTE_OF_HOUR, 1, 2, SignStyle.NEVER) + .appendLiteral(':') + .appendValue(ChronoField.SECOND_OF_MINUTE, 1, 2, SignStyle.NEVER) + .optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 6, true) + .optionalStart() + .toFormatter() + + // DATE + // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#date_type + // YYYY-[M]M-[D]D + private val dateFormatter = new DateTimeFormatterBuilder() + .appendValue(ChronoField.YEAR, 4) + .appendLiteral('-') + .appendValue(ChronoField.MONTH_OF_YEAR, 1, 2, SignStyle.NEVER) + .appendLiteral('-') + .appendValue(ChronoField.DAY_OF_MONTH, 1, 2, SignStyle.NEVER) + .toFormatter() + // TIMESTAMP - // YYYY-[M]M-[D]D[ [H]H:[M]M:[S]S[.DDDDDD]][time zone] - private val timestampFormatter = + // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#time_type + + // civil_date_part YYYY-[M]M-[D]D + private val civilDatePartFormatter = dateFormatter + // time_part { |T|t}[H]H:[M]M:[S]S[.F] + private val timePartFormatter = new DateTimeFormatterBuilder() - .parseLenient() - .append(DateTimeFormatter.ISO_LOCAL_DATE) - .appendLiteral(' ') - .appendValue(ChronoField.HOUR_OF_DAY, 2) - .appendLiteral(':') - .appendValue(ChronoField.MINUTE_OF_HOUR, 2) - .optionalStart() - .appendLiteral(':') - .appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .padNext(1) .optionalStart() - .appendFraction(ChronoField.NANO_OF_SECOND, 6, 9, true) - .optionalStart() - .appendOffset("+HHMM", "+00:00") + .parseCaseInsensitive() + .appendLiteral('T') + .parseCaseSensitive() .optionalEnd() - .toFormatter() - .withZone(ZoneOffset.UTC) - private val timestampValidator = + .append(timeFormatter) + .toFormatter + + // time_zone_offset or utc_time_zone + // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#time_zones + // {+|-}H[H][:M[M]] or {Z|z} + private val timeZoneOffsetFormatter = new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .appendOffsetId() + .parseCaseSensitive() + .toFormatter + + // time_zone + private val timeZoneFormatter = new DateTimeFormatterBuilder() + .appendZoneRegionId() + .toFormatter + + // timestamp + // { + // civil_date_part[time_part [time_zone]] | + // civil_date_part[time_part[time_zone_offset]] | + // civil_date_part[time_part[utc_time_zone]] + // } + private val timestampFormatter = new DateTimeFormatterBuilder() - .parseLenient() - .append(timestampFormatter) + .append(civilDatePartFormatter) + .append(timePartFormatter) .optionalStart() - .appendOffsetId() + .append(timeZoneOffsetFormatter) + .optionalEnd() + .optionalStart() + .appendLiteral(' ') + .append(timeZoneFormatter) .optionalEnd() - .toFormatter() + .toFormatter .withZone(ZoneOffset.UTC) - // DATE - // YYYY-[M]M-[D]D - private val dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd") - - // TIME - // [H]H:[M]M:[S]S[.DDDDDD] - private val timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS") - // DATETIME - // YYYY-[M]M-[D]D[ [H]H:[M]M:[S]S[.DDDDDD]] - private val datetimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS"); + // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#datetime_type + // civil_date_part[time_part] + private val datetimeFormatter = new DateTimeFormatterBuilder() + .append(civilDatePartFormatter) + .appendOptional(timePartFormatter) + .toFormatter - def toInstant(v: Any): Instant = Instant.from(timestampValidator.parse(v.toString)) - def fromInstant(i: Instant): Any = timestampFormatter.format(i) + def toInstant(v: Any): Instant = Instant.from(timestampFormatter.parse(v.toString)) + def fromInstant(i: Instant): Any = DateTimeFormatter.ISO_INSTANT.format(i) def toLocalDate(v: Any): LocalDate = LocalDate.from(dateFormatter.parse(v.toString)) - def fromLocalDate(d: LocalDate): Any = dateFormatter.format(d) + def fromLocalDate(d: LocalDate): Any = DateTimeFormatter.ISO_LOCAL_DATE.format(d) def toLocalTime(v: Any): LocalTime = LocalTime.from(timeFormatter.parse(v.toString)) - def fromLocalTime(t: LocalTime): Any = timeFormatter.format(t) + def fromLocalTime(t: LocalTime): Any = DateTimeFormatter.ISO_LOCAL_TIME.format(t) def toLocalDateTime(v: Any): LocalDateTime = LocalDateTime.from(datetimeFormatter.parse(v.toString)) - def fromLocalDateTime(dt: LocalDateTime): Any = datetimeFormatter.format(dt) + def fromLocalDateTime(dt: LocalDateTime): Any = DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(dt) } diff --git a/bigquery/src/test/scala/magnolify/bigquery/TimestampConverterSuite.scala b/bigquery/src/test/scala/magnolify/bigquery/TimestampConverterSuite.scala new file mode 100644 index 000000000..61aa06836 --- /dev/null +++ b/bigquery/src/test/scala/magnolify/bigquery/TimestampConverterSuite.scala @@ -0,0 +1,106 @@ +package magnolify.bigquery + +import scala.util.control.NonFatal + +class TimestampConverterSuite extends munit.ScalaCheckSuite { + + test("TIMESTAMP") { + Seq( + // this should be the default format sent by BQ API + "2023-01-01 10:11:12.123456 UTC", + // optional digits + "2023-01-01 10:11:12.123456", + "2023-01-01 10:11:12.1234", + "2023-01-01 10:11:12", + "2023-1-1 0:1:2", + // time separator + "2023-01-01T10:11:12.123456 UTC", + "2023-01-01t10:11:12.123456 UTC", + // offsets + "2023-01-01t10:11:12.123456Z", + "2023-01-01t10:11:12.123456z", + "2023-01-01t10:11:12.123456+01:02", + "2023-01-01t10:11:12.123456-01:02", + // "2023-01-01t10:11:12.123456+1:2", not possible to get +H:M offset pattern + // "2023-01-01t10:11:12.123456+1", skipped + // zone + "2023-01-01t10:11:12.123456 Europe/London", + "2023-01-01t10:11:12.123456 UTC+01:02", + "2023-01-01t10:11:12.123456 UTC-01:02", + // "2023-01-01t10:11:12.123456 UTC+1:2", not possible to get +H:M offset pattern + // "2023-01-01t10:11:12.123456 UTC+1", skipped + ).foreach { ts => + try { + TimestampConverter.toInstant(ts) + } catch { + case NonFatal(e) => throw new Exception(s"Failed parsing $ts", e) + } + } + // formatter uses ISO + val javaInstant = TimestampConverter.toInstant("2023-01-01 10:11:12.123456 UTC") + val actual = TimestampConverter.fromInstant(javaInstant) + assertEquals(actual, "2023-01-01T10:11:12.123456Z") + } + + test("DATE") { + Seq( + "2023-01-02", + "2023-01-2", + "2023-1-2", + ).foreach { date => + try { + TimestampConverter.toLocalDate(date) + } catch { + case NonFatal(e) => throw new Exception(s"Failed parsing $date", e) + } + } + // formatter uses ISO + val javaLocalDate = TimestampConverter.toLocalDate("2023-01-02") + val actual = TimestampConverter.fromLocalDate(javaLocalDate) + assertEquals(actual, "2023-01-02") + } + + test("TIME") { + Seq( + "10:11:12.123456", + "10:11:12.1234", + "10:11:12", + "0:1:2", + "0:1:2.123456" + ).foreach { t => + try { + TimestampConverter.toLocalTime(t) + } catch { + case NonFatal(e) => throw new Exception(s"Failed parsing $t", e) + } + } + // formatter uses ISO + val javaLocalTime = TimestampConverter.toLocalTime("16:03:57.029881") + val actual = TimestampConverter.fromLocalTime(javaLocalTime) + assertEquals(actual, "16:03:57.029881") + } + + test("DATETIME") { + Seq( + "2023-01-01 10:11:12.123456", + // optional digits + "2023-01-01 10:11:12.123456", + "2023-01-01 10:11:12.1234", + "2023-01-01 10:11:12", + "2023-1-1 0:1:2", + // time separator + "2023-01-01T10:11:12.123456", + "2023-01-01t10:11:12.123456", + ).foreach { datetime => + try { + TimestampConverter.toLocalDateTime(datetime) + } catch { + case NonFatal(e) => throw new Exception(s"Failed parsing $datetime", e) + } + } + // formatter uses ISO + val javaLocalDateTime = TimestampConverter.toLocalDateTime("2023-01-01 10:11:12.123456") + val actual = TimestampConverter.fromLocalDateTime(javaLocalDateTime) + assertEquals(actual, "2023-01-01T10:11:12.123456") + } +}