Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Break backward compatibility #584

Open
zeminzhou opened this issue May 22, 2024 · 1 comment
Open

Break backward compatibility #584

zeminzhou opened this issue May 22, 2024 · 1 comment

Comments

@zeminzhou
Copy link

zeminzhou commented May 22, 2024

Test code

package main

import (
        "testing"

        "github.com/stretchr/testify/require"
        "github.com/xitongsys/parquet-go-source/local"
        "github.com/xitongsys/parquet-go/reader"
        "github.com/xitongsys/parquet-go/writer"
)

func TestParquetVariousTypes(t *testing.T) {
        type Test struct {
                Date            int32 `parquet:"name=date, type=INT32, convertedtype=date"`
                TimeMillis      int32 `parquet:"name=timemillis, type=INT32, convertedtype=TIME_MILLIS"`
                TimeMicros      int64 `parquet:"name=timemicros, type=INT64, convertedtype=TIME_MICROS"`
                TimestampMillis int64 `parquet:"name=timestampmillis, type=INT64, convertedtype=TIMESTAMP_MILLIS"`
                TimestampMicros int64 `parquet:"name=timestampmicros, type=INT64, convertedtype=TIMESTAMP_MICROS"`

                Decimal1 int32 `parquet:"name=decimal1, type=INT32, convertedtype=DECIMAL, scale=2, precision=9"`
                Decimal2 int32 `parquet:"name=decimal2, type=INT32, convertedtype=DECIMAL, scale=4, precision=4"`
                Decimal3 int64 `parquet:"name=decimal3, type=INT64, convertedtype=DECIMAL, scale=2, precision=18"`
                Decimal6 int32 `parquet:"name=decimal6, type=INT32, convertedtype=DECIMAL, scale=4, precision=4"`
        }

        // prepare data
        name := "test123.parquet"
        fw, err := local.NewLocalFileWriter(name)
        require.NoError(t, err)
        test := &Test{}
        writer0, err := writer.NewParquetWriter(fw, test, 2)
        require.NoError(t, err)

        v := &Test{
                Date:            18564,              // 2020-10-29
                TimeMillis:      62775123,           // 17:26:15.123
                TimeMicros:      62775123456,        // 17:26:15.123
                TimestampMillis: 1603963672356,      // 2020-10-29T09:27:52.356Z
                TimestampMicros: 1603963672356956,   // 2020-10-29T09:27:52.356956Z
                Decimal1:        -12345678,          // -123456.78
                Decimal2:        456,                // 0.0456
                Decimal3:        123456789012345678, // 1234567890123456.78
                Decimal6:        -1,                 // -0.0001
        }
        require.NoError(t, writer0.Write(v))
        require.NoError(t, writer0.WriteStop())
        require.NoError(t, fw.Close())

        fr, err := local.NewLocalFileReader(name)
        require.NoError(t, err)
        reader, err := reader.NewParquetReader(fr, nil, 2)
        require.NoError(t, err)

        for _, c := range reader.SchemaHandler.SchemaElements {
                if c != nil && c.LogicalType != nil {
                        if c.LogicalType.IsSetTIMESTAMP() {
                                require.True(t, c.LogicalType.TIMESTAMP.GetIsAdjustedToUTC())
                        } else if c.LogicalType.IsSetTIME() {
                                require.True(t, c.LogicalType.TIME.GetIsAdjustedToUTC())
                        }
                }
        }
}

From document, the GetIsAdjustedToUTC from return true. https://github.com/apache/parquet-format/blob/master/LogicalTypes.md

@zeminzhou
Copy link
Author

/cc @xitongsys

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

1 participant