Merge pull request #145 from apstndb/migrate-to-memefish

[v2] Migrate to memefish

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 jobs:
   test:
     docker:
-      - image: golang:1.21-bullseye
+      - image: golang:1.22-bullseye
         environment:
           SPANNER_EMULATOR_HOST: localhost:9010
           SPANNER_EMULATOR_HOST_REST: localhost:9020
@@ -55,7 +55,7 @@ jobs:
 
   v2test:
     docker:
-      - image: golang:1.21-bullseye
+      - image: golang:1.22-bullseye
         environment:
           SPANNER_EMULATOR_HOST: localhost:9010
           SPANNER_EMULATOR_HOST_REST: localhost:9020
@@ -100,7 +100,7 @@ jobs:
             git diff --quiet v2/test/
   check_lint:
     docker:
-      - image: golang:1.21-bullseye
+      - image: golang:1.22-bullseye
     working_directory: /go/src/github.com/cloudspannerecosystem/yo
     steps:
       - checkout
@@ -110,7 +110,7 @@ jobs:
             make check_lint
   check_go_mod:
     docker:
-      - image: golang:1.21-bullseye
+      - image: golang:1.22-bullseye
     working_directory: /go/src/github.com/cloudspannerecosystem/yo
     steps:
       - checkout

v2/README.md

@@ -661,7 +661,6 @@ inflections:
    * The original function name based on index column names is ambiguous if there are multiple index that use the same index columns
    * The naming rule for the new function names is `Find` + _TABLE_NAME_ + _INDEX_NAME
    * Use `--use-legacy-index-module` option if you still want to use function names based on index column names
-* Use spansql package instead of memefish to parse DDL statements
 * Generated filenames become snake_case names
 
 ### Deprecations

v2/go.mod

@@ -1,11 +1,14 @@
 module go.mercari.io/yo/v2
 
-go 1.21
+go 1.22.0
+
+toolchain go1.23.2
 
 require (
 	cloud.google.com/go v0.110.7
 	cloud.google.com/go/spanner v1.48.0
-	github.com/google/go-cmp v0.5.9
+	github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7
+	github.com/google/go-cmp v0.6.0
 	github.com/googleapis/gax-go/v2 v2.12.0
 	github.com/jinzhu/inflection v1.0.0
 	github.com/kenshaw/snaker v0.2.0
@@ -42,7 +45,7 @@ require (
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

v2/go.sum

@@ -13,6 +13,8 @@ cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHS
 cloud.google.com/go/spanner v1.48.0 h1:lh3Xqe2G+/bhJ1O3JxYt4ahYXOz/wPH4D2Wrx2vFoNI=
 cloud.google.com/go/spanner v1.48.0/go.mod h1:eGj9mQGK8+hkgSVbHNQ06pQ4oS+cyc4tXXd6Dif1KoM=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=
+github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=
@@ -21,6 +23,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7 h1:vmS9Nvh7ij5EVnMwvkWsL84xzx5cGM3j/SJlM1zfVLE=
+github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7/go.mod h1:iYAaNZfVIn4QYfUmXt+3EeHAok/kqpN/fp/8kgDHjx8=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
@@ -73,8 +77,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg=
 github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -89,8 +93,15 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc h1:XLjmW07gT7cG/wb6mavIrvAIWBYaTacPo8UOnxGSspA=
+github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc/go.mod h1:qK2ivXw91omfE1uXcpR5kWbAMZRdDOnGbqWlZ7reRFk=
 github.com/kenshaw/snaker v0.2.0 h1:DPlxCtAv9mw1wSsvIN1khUAPJUIbFJUckMIDWSQ7TC8=
 github.com/kenshaw/snaker v0.2.0/go.mod h1:DNyRUqHMZ18/zioxr6R7m4kSxxf2+QmB0BXoORsXRaY=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@@ -160,8 +171,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

v2/loader/schema_parser_source.go

@@ -25,9 +25,17 @@ import (
 	"sort"
 	"strings"
 
-	"cloud.google.com/go/spanner/spansql"
+	"github.com/cloudspannerecosystem/memefish"
+	"github.com/cloudspannerecosystem/memefish/ast"
 )
 
+func extractName(path *ast.Path) (string, error) {
+	if len(path.Idents) != 1 {
+		return "", fmt.Errorf("path isn't simple ident: %v", path.SQL())
+	}
+	return path.Idents[0].Name, nil
+}
+
 func NewSchemaParserSource(fpath string) (SchemaSource, error) {
 	b, err := os.ReadFile(fpath)
 	if err != nil {
@@ -42,23 +50,31 @@ func NewSchemaParserSource(fpath string) (SchemaSource, error) {
 			continue
 		}
 
-		ddlstmt, err := spansql.ParseDDLStmt(stmt)
+		ddlstmt, err := memefish.ParseDDL("", stmt)
 		if err != nil {
 			return nil, err
 		}
 
 		switch val := ddlstmt.(type) {
-		case *spansql.CreateTable:
-			tableName := string(val.Name)
+		case *ast.CreateTable:
+			tableName, err := extractName(val.Name)
+			if err != nil {
+				return nil, err
+			}
+
 			v := tables[tableName]
 			v.createTable = val
 			tables[tableName] = v
-		case *spansql.CreateIndex:
-			tableName := string(val.Table)
+		case *ast.CreateIndex:
+			tableName, err := extractName(val.TableName)
+			if err != nil {
+				return nil, err
+			}
+
 			v := tables[tableName]
 			v.createIndexes = append(v.createIndexes, val)
 			tables[tableName] = v
-		case *spansql.AlterTable:
+		case *ast.AlterTable:
 			if isAlterTableAddFK(val) {
 				continue
 			}
@@ -69,18 +85,18 @@ func NewSchemaParserSource(fpath string) (SchemaSource, error) {
 	return &schemaParserSource{tables: tables}, nil
 }
 
-func isAlterTableAddFK(at *spansql.AlterTable) bool {
-	ac, ok := at.Alteration.(spansql.AddConstraint)
+func isAlterTableAddFK(at *ast.AlterTable) bool {
+	ac, ok := at.TableAlteration.(*ast.AddTableConstraint)
 	if !ok {
 		return false
 	}
-	_, ok = ac.Constraint.Constraint.(spansql.ForeignKey)
+	_, ok = ac.TableConstraint.Constraint.(*ast.ForeignKey)
 	return ok
 }
 
 type table struct {
-	createTable   *spansql.CreateTable
-	createIndexes []*spansql.CreateIndex
+	createTable   *ast.CreateTable
+	createIndexes []*ast.CreateIndex
 }
 
 type schemaParserSource struct {
@@ -91,12 +107,20 @@ func (s *schemaParserSource) TableList() ([]*SpannerTable, error) {
 	var tables []*SpannerTable
 	for _, t := range s.tables {
 		var parent string
-		if t.createTable.Interleave != nil {
-			parent = string(t.createTable.Interleave.Parent)
+		if t.createTable.Cluster != nil {
+			var err error
+			parent, err = extractName(t.createTable.Cluster.TableName)
+			if err != nil {
+				return nil, err
+			}
+		}
+		tableName, err := extractName(t.createTable.Name)
+		if err != nil {
+			return nil, err
 		}
 
 		tables = append(tables, &SpannerTable{
-			TableName:       string(t.createTable.Name),
+			TableName:       tableName,
 			ParentTableName: parent,
 		})
 	}
@@ -113,19 +137,19 @@ func (s *schemaParserSource) ColumnList(name string) ([]*SpannerColumn, error) {
 	table := s.tables[name].createTable
 
 	check := make(map[string]struct{})
-	for _, pk := range table.PrimaryKey {
-		check[string(pk.Column)] = struct{}{}
+	for _, pk := range table.PrimaryKeys {
+		check[pk.Name.Name] = struct{}{}
 	}
 
 	for i, c := range table.Columns {
-		_, pk := check[string(c.Name)]
+		_, pk := check[c.Name.Name]
 		cols = append(cols, &SpannerColumn{
 			FieldOrdinal: i + 1,
-			ColumnName:   string(c.Name),
+			ColumnName:   c.Name.Name,
 			DataType:     c.Type.SQL(),
 			NotNull:      c.NotNull,
 			IsPrimaryKey: pk,
-			IsGenerated:  c.Generated != nil,
+			IsGenerated:  c.GeneratedExpr != nil,
 		})
 	}
 
@@ -135,8 +159,13 @@ func (s *schemaParserSource) ColumnList(name string) ([]*SpannerColumn, error) {
 func (s *schemaParserSource) IndexList(name string) ([]*SpannerIndex, error) {
 	var indexes []*SpannerIndex
 	for _, index := range s.tables[name].createIndexes {
+		indexName, err := extractName(index.Name)
+		if err != nil {
+			return nil, err
+		}
+
 		indexes = append(indexes, &SpannerIndex{
-			IndexName: string(index.Name),
+			IndexName: indexName,
 			IsUnique:  index.Unique,
 		})
 	}
@@ -151,23 +180,29 @@ func (s *schemaParserSource) IndexColumnList(table, index string) ([]*SpannerInd
 
 	var cols []*SpannerIndexColumn
 	for _, ix := range s.tables[table].createIndexes {
-		if string(ix.Name) != index {
+		ixName, err := extractName(ix.Name)
+		if err != nil {
+			return nil, err
+		}
+		if ixName != index {
 			continue
 		}
 
-		// add storing columns first
-		for _, storing := range ix.Storing {
-			cols = append(cols, &SpannerIndexColumn{
-				SeqNo:      0,
-				Storing:    true,
-				ColumnName: string(storing),
-			})
+		if ix.Storing != nil {
+			// add storing columns first
+			for _, storing := range ix.Storing.Columns {
+				cols = append(cols, &SpannerIndexColumn{
+					SeqNo:      0,
+					Storing:    true,
+					ColumnName: storing.Name,
+				})
+			}
 		}
 
-		for i, c := range ix.Columns {
+		for i, c := range ix.Keys {
 			cols = append(cols, &SpannerIndexColumn{
 				SeqNo:      i + 1,
-				ColumnName: string(c.Column),
+				ColumnName: c.Name.Name,
 			})
 		}
 		break
@@ -183,10 +218,10 @@ func (s *schemaParserSource) primaryKeyColumnList(table string) ([]*SpannerIndex
 	}
 
 	var cols []*SpannerIndexColumn
-	for i, key := range tbl.createTable.PrimaryKey {
+	for i, key := range tbl.createTable.PrimaryKeys {
 		cols = append(cols, &SpannerIndexColumn{
 			SeqNo:      i + 1,
-			ColumnName: string(key.Column),
+			ColumnName: key.Name.Name,
 		})
 	}