From 412346ea78fef62e3e1cdb7928bef1c222664576 Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Tue, 22 Oct 2024 14:30:06 +0000 Subject: [PATCH 1/3] Use query builders for DML event queries --- go/logic/applier.go | 52 ++++++++++-- go/logic/applier_test.go | 7 ++ go/logic/migrator.go | 4 + go/sql/builder.go | 168 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 223 insertions(+), 8 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 990fbe720..f74af8762 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -60,6 +60,10 @@ type Applier struct { migrationContext *base.MigrationContext finishedMigrating int64 name string + + dmlDeleteQueryBuilder *sql.DMLDeleteQueryBuilder + dmlInsertQueryBuilder *sql.DMLInsertQueryBuilder + dmlUpdateQueryBuilder *sql.DMLUpdateQueryBuilder } func NewApplier(migrationContext *base.MigrationContext) *Applier { @@ -106,6 +110,37 @@ func (this *Applier) InitDBConnections() (err error) { return nil } +func (this *Applier) prepareQueries() (err error) { + if this.dmlDeleteQueryBuilder, err = sql.NewDMLDeleteQueryBuilder( + this.migrationContext.DatabaseName, + this.migrationContext.GetGhostTableName(), + this.migrationContext.OriginalTableColumns, + &this.migrationContext.UniqueKey.Columns, + ); err != nil { + return err + } + if this.dmlInsertQueryBuilder, err = sql.NewDMLInsertQueryBuilder( + this.migrationContext.DatabaseName, + this.migrationContext.GetGhostTableName(), + this.migrationContext.OriginalTableColumns, + this.migrationContext.SharedColumns, + this.migrationContext.MappedSharedColumns, + ); err != nil { + return err + } + if this.dmlUpdateQueryBuilder, err = sql.NewDMLUpdateQueryBuilder( + this.migrationContext.DatabaseName, + this.migrationContext.GetGhostTableName(), + this.migrationContext.OriginalTableColumns, + this.migrationContext.SharedColumns, + this.migrationContext.MappedSharedColumns, + &this.migrationContext.UniqueKey.Columns, + ); err != nil { + return err + } + return nil +} + // validateAndReadGlobalVariables potentially reads server global variables, such as the time_zone and wait_timeout. func (this *Applier) validateAndReadGlobalVariables() error { query := `select /* gh-ost */ @@global.time_zone, @@global.wait_timeout` @@ -1135,35 +1170,36 @@ func (this *Applier) updateModifiesUniqueKeyColumns(dmlEvent *binlog.BinlogDMLEv // buildDMLEventQuery creates a query to operate on the ghost table, based on an intercepted binlog // event entry on the original table. -func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (results [](*dmlBuildResult)) { +func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) []*dmlBuildResult { switch dmlEvent.DML { case binlog.DeleteDML: { - query, uniqueKeyArgs, err := sql.BuildDMLDeleteQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.WhereColumnValues.AbstractValues()) - return append(results, newDmlBuildResult(query, uniqueKeyArgs, -1, err)) + query, uniqueKeyArgs, err := this.dmlDeleteQueryBuilder.BuildQuery(dmlEvent.WhereColumnValues.AbstractValues()) + return []*dmlBuildResult{newDmlBuildResult(query, uniqueKeyArgs, -1, err)} } case binlog.InsertDML: { - query, sharedArgs, err := sql.BuildDMLInsertQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, dmlEvent.NewColumnValues.AbstractValues()) - return append(results, newDmlBuildResult(query, sharedArgs, 1, err)) + query, sharedArgs, err := this.dmlInsertQueryBuilder.BuildQuery(dmlEvent.NewColumnValues.AbstractValues()) + return []*dmlBuildResult{newDmlBuildResult(query, sharedArgs, 1, err)} } case binlog.UpdateDML: { if _, isModified := this.updateModifiesUniqueKeyColumns(dmlEvent); isModified { + results := make([]*dmlBuildResult, 0, 2) dmlEvent.DML = binlog.DeleteDML results = append(results, this.buildDMLEventQuery(dmlEvent)...) dmlEvent.DML = binlog.InsertDML results = append(results, this.buildDMLEventQuery(dmlEvent)...) return results } - query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues()) + query, sharedArgs, uniqueKeyArgs, err := this.dmlUpdateQueryBuilder.BuildQuery(dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues()) args := sqlutils.Args() args = append(args, sharedArgs...) args = append(args, uniqueKeyArgs...) - return append(results, newDmlBuildResult(query, args, 0, err)) + return []*dmlBuildResult{newDmlBuildResult(query, args, 0, err)} } } - return append(results, newDmlBuildResultError(fmt.Errorf("Unknown dml event type: %+v", dmlEvent.DML))) + return []*dmlBuildResult{newDmlBuildResultError(fmt.Errorf("Unknown dml event type: %+v", dmlEvent.DML))} } // ApplyDMLEventQueries applies multiple DML queries onto the _ghost_ table diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index e9426bbc5..d17462781 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -100,6 +100,7 @@ func TestApplierBuildDMLEventQuery(t *testing.T) { columnValues := sql.ToColumnValues([]interface{}{123456, 42}) migrationContext := base.NewMigrationContext() + migrationContext.DatabaseName = "test" migrationContext.OriginalTableName = "test" migrationContext.OriginalTableColumns = columns migrationContext.SharedColumns = columns @@ -110,6 +111,7 @@ func TestApplierBuildDMLEventQuery(t *testing.T) { } applier := NewApplier(migrationContext) + applier.prepareQueries() t.Run("delete", func(t *testing.T) { binlogEvent := &binlog.BinlogDMLEvent{ @@ -290,8 +292,13 @@ func (suite *ApplierTestSuite) TestApplyDMLEventQueries() { migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "item_id"}) migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "primary_key", + Columns: *sql.NewColumnList([]string{"id"}), + } applier := NewApplier(migrationContext) + suite.Require().NoError(applier.prepareQueries()) defer applier.Teardown() err = applier.InitDBConnections() diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 3fc897cd3..2996ac642 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -386,6 +386,10 @@ func (this *Migrator) Migrate() (err error) { if err := this.inspector.inspectOriginalAndGhostTables(); err != nil { return err } + // We can prepare some of the queries on the applier + if err := this.applier.prepareQueries(); err != nil { + return err + } // Validation complete! We're good to execute this migration if err := this.hooksExecutor.onValidated(); err != nil { return err diff --git a/go/sql/builder.go b/go/sql/builder.go index 7be428f93..6fd4a85d8 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -530,3 +530,171 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol ) return result, sharedArgs, uniqueKeyArgs, nil } + +type DMLDeleteQueryBuilder struct { + tableColumns, uniqueKeyColumns *ColumnList + preparedStatement string +} + +func NewDMLDeleteQueryBuilder(databaseName, tableName string, tableColumns, uniqueKeyColumns *ColumnList) (*DMLDeleteQueryBuilder, error) { + if uniqueKeyColumns.Len() == 0 { + return nil, fmt.Errorf("no unique key columns found in NewDMLDeleteQueryBuilder") + } + databaseName = EscapeName(databaseName) + tableName = EscapeName(tableName) + equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names()) + if err != nil { + return nil, err + } + + stmt := fmt.Sprintf(` + delete /* gh-ost %s.%s */ + from + %s.%s + where + %s`, + databaseName, tableName, + databaseName, tableName, + equalsComparison, + ) + + b := &DMLDeleteQueryBuilder{ + tableColumns: tableColumns, + uniqueKeyColumns: uniqueKeyColumns, + preparedStatement: stmt, + } + return b, nil +} + +func (b *DMLDeleteQueryBuilder) BuildQuery(args []interface{}) (string, []interface{}, error) { + if len(args) != b.tableColumns.Len() { + return "", nil, fmt.Errorf("args count differs from table column count in BuildDMLDeleteQuery") + } + uniqueKeyArgs := make([]interface{}, 0, b.uniqueKeyColumns.Len()) + for _, column := range b.uniqueKeyColumns.Columns() { + tableOrdinal := b.tableColumns.Ordinals[column.Name] + arg := column.convertArg(args[tableOrdinal], true) + uniqueKeyArgs = append(uniqueKeyArgs, arg) + } + return b.preparedStatement, uniqueKeyArgs, nil +} + +type DMLInsertQueryBuilder struct { + tableColumns, sharedColumns *ColumnList + preparedStatement string +} + +func NewDMLInsertQueryBuilder(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns *ColumnList) (*DMLInsertQueryBuilder, error) { + if !sharedColumns.IsSubsetOf(tableColumns) { + return nil, fmt.Errorf("shared columns is not a subset of table columns in NewDMLInsertQueryBuilder") + } + if sharedColumns.Len() == 0 { + return nil, fmt.Errorf("no shared columns found in NewDMLInsertQueryBuilder") + } + databaseName = EscapeName(databaseName) + tableName = EscapeName(tableName) + mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names()) + for i := range mappedSharedColumnNames { + mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i]) + } + preparedValues := buildColumnsPreparedValues(mappedSharedColumns) + + stmt := fmt.Sprintf(` + replace /* gh-ost %s.%s */ + into + %s.%s + (%s) + values + (%s)`, + databaseName, tableName, + databaseName, tableName, + strings.Join(mappedSharedColumnNames, ", "), + strings.Join(preparedValues, ", "), + ) + + return &DMLInsertQueryBuilder{ + tableColumns: tableColumns, + sharedColumns: sharedColumns, + preparedStatement: stmt, + }, nil +} + +func (b *DMLInsertQueryBuilder) BuildQuery(args []interface{}) (string, []interface{}, error) { + if len(args) != b.tableColumns.Len() { + return "", nil, fmt.Errorf("args count differs from table column count in BuildDMLInsertQuery") + } + sharedArgs := make([]interface{}, 0, b.sharedColumns.Len()) + for _, column := range b.sharedColumns.Columns() { + tableOrdinal := b.tableColumns.Ordinals[column.Name] + arg := column.convertArg(args[tableOrdinal], false) + sharedArgs = append(sharedArgs, arg) + } + return b.preparedStatement, sharedArgs, nil +} + +type DMLUpdateQueryBuilder struct { + tableColumns, sharedColumns, uniqueKeyColumns *ColumnList + preparedStatement string +} + +func NewDMLUpdateQueryBuilder(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns, uniqueKeyColumns *ColumnList) (*DMLUpdateQueryBuilder, error) { + if !sharedColumns.IsSubsetOf(tableColumns) { + return nil, fmt.Errorf("shared columns is not a subset of table columns in NewDMLUpdateQueryBuilder") + } + if !uniqueKeyColumns.IsSubsetOf(sharedColumns) { + return nil, fmt.Errorf("unique key columns is not a subset of shared columns in NewDMLUpdateQueryBuilder") + } + if sharedColumns.Len() == 0 { + return nil, fmt.Errorf("no shared columns found in NewDMLUpdateQueryBuilder") + } + if uniqueKeyColumns.Len() == 0 { + return nil, fmt.Errorf("no unique key columns found in NewDMLUpdateQueryBuilder") + } + databaseName = EscapeName(databaseName) + tableName = EscapeName(tableName) + setClause, err := BuildSetPreparedClause(mappedSharedColumns) + if err != nil { + return nil, err + } + + equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names()) + if err != nil { + return nil, err + } + stmt := fmt.Sprintf(` + update /* gh-ost %s.%s */ + %s.%s + set + %s + where + %s`, + databaseName, tableName, + databaseName, tableName, + setClause, + equalsComparison, + ) + return &DMLUpdateQueryBuilder{ + tableColumns: tableColumns, + sharedColumns: sharedColumns, + uniqueKeyColumns: uniqueKeyColumns, + preparedStatement: stmt, + }, nil +} + +func (b *DMLUpdateQueryBuilder) BuildQuery(valueArgs, whereArgs []interface{}) (string, []interface{}, []interface{}, error) { + sharedArgs := make([]interface{}, 0, b.sharedColumns.Len()) + for _, column := range b.sharedColumns.Columns() { + tableOrdinal := b.tableColumns.Ordinals[column.Name] + arg := column.convertArg(valueArgs[tableOrdinal], false) + sharedArgs = append(sharedArgs, arg) + } + + uniqueKeyArgs := make([]interface{}, 0, b.uniqueKeyColumns.Len()) + for _, column := range b.uniqueKeyColumns.Columns() { + tableOrdinal := b.tableColumns.Ordinals[column.Name] + arg := column.convertArg(whereArgs[tableOrdinal], true) + uniqueKeyArgs = append(uniqueKeyArgs, arg) + } + + return b.preparedStatement, sharedArgs, uniqueKeyArgs, nil +} From 4608dcbcab8fbfb04c8bebc4a68e384d8a73c550 Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Wed, 23 Oct 2024 08:43:04 +0000 Subject: [PATCH 2/3] Temporarily moving one check on column lists to `DMLUpdateQueryBuilder.BuildQuery()` --- go/sql/builder.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/go/sql/builder.go b/go/sql/builder.go index 6fd4a85d8..5acc1a2aa 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -641,9 +641,6 @@ func NewDMLUpdateQueryBuilder(databaseName, tableName string, tableColumns, shar if !sharedColumns.IsSubsetOf(tableColumns) { return nil, fmt.Errorf("shared columns is not a subset of table columns in NewDMLUpdateQueryBuilder") } - if !uniqueKeyColumns.IsSubsetOf(sharedColumns) { - return nil, fmt.Errorf("unique key columns is not a subset of shared columns in NewDMLUpdateQueryBuilder") - } if sharedColumns.Len() == 0 { return nil, fmt.Errorf("no shared columns found in NewDMLUpdateQueryBuilder") } @@ -682,6 +679,11 @@ func NewDMLUpdateQueryBuilder(databaseName, tableName string, tableColumns, shar } func (b *DMLUpdateQueryBuilder) BuildQuery(valueArgs, whereArgs []interface{}) (string, []interface{}, []interface{}, error) { + // TODO: move this check back to `NewDMLUpdateQueryBuilder()`, needs fix on generated columns. + if !b.uniqueKeyColumns.IsSubsetOf(b.sharedColumns) { + return "", nil, nil, fmt.Errorf("unique key columns is not a subset of shared columns in DMLUpdateQueryBuilder") + } + sharedArgs := make([]interface{}, 0, b.sharedColumns.Len()) for _, column := range b.sharedColumns.Columns() { tableOrdinal := b.tableColumns.Ordinals[column.Name] From ab74e9927c63bc4c4c5f63fb0e13504de1f4162b Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Wed, 23 Oct 2024 11:38:32 +0000 Subject: [PATCH 3/3] Add doc comments and remove old DML query builder functions --- go/sql/builder.go | 155 +++++++---------------------------------- go/sql/builder_test.go | 78 +++++++++++++++------ 2 files changed, 82 insertions(+), 151 deletions(-) diff --git a/go/sql/builder.go b/go/sql/builder.go index 5acc1a2aa..a3e702b48 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -402,140 +402,17 @@ func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName string, uni return query, nil } -func BuildDMLDeleteQuery(databaseName, tableName string, tableColumns, uniqueKeyColumns *ColumnList, args []interface{}) (result string, uniqueKeyArgs []interface{}, err error) { - if len(args) != tableColumns.Len() { - return result, uniqueKeyArgs, fmt.Errorf("args count differs from table column count in BuildDMLDeleteQuery") - } - if uniqueKeyColumns.Len() == 0 { - return result, uniqueKeyArgs, fmt.Errorf("No unique key columns found in BuildDMLDeleteQuery") - } - for _, column := range uniqueKeyColumns.Columns() { - tableOrdinal := tableColumns.Ordinals[column.Name] - arg := column.convertArg(args[tableOrdinal], true) - uniqueKeyArgs = append(uniqueKeyArgs, arg) - } - databaseName = EscapeName(databaseName) - tableName = EscapeName(tableName) - equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names()) - if err != nil { - return result, uniqueKeyArgs, err - } - result = fmt.Sprintf(` - delete /* gh-ost %s.%s */ - from - %s.%s - where - %s`, - databaseName, tableName, - databaseName, tableName, - equalsComparison, - ) - return result, uniqueKeyArgs, nil -} - -func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns *ColumnList, args []interface{}) (result string, sharedArgs []interface{}, err error) { - if len(args) != tableColumns.Len() { - return result, args, fmt.Errorf("args count differs from table column count in BuildDMLInsertQuery") - } - if !sharedColumns.IsSubsetOf(tableColumns) { - return result, args, fmt.Errorf("shared columns is not a subset of table columns in BuildDMLInsertQuery") - } - if sharedColumns.Len() == 0 { - return result, args, fmt.Errorf("No shared columns found in BuildDMLInsertQuery") - } - databaseName = EscapeName(databaseName) - tableName = EscapeName(tableName) - - for _, column := range sharedColumns.Columns() { - tableOrdinal := tableColumns.Ordinals[column.Name] - arg := column.convertArg(args[tableOrdinal], false) - sharedArgs = append(sharedArgs, arg) - } - - mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names()) - for i := range mappedSharedColumnNames { - mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i]) - } - preparedValues := buildColumnsPreparedValues(mappedSharedColumns) - - result = fmt.Sprintf(` - replace /* gh-ost %s.%s */ - into - %s.%s - (%s) - values - (%s)`, - databaseName, tableName, - databaseName, tableName, - strings.Join(mappedSharedColumnNames, ", "), - strings.Join(preparedValues, ", "), - ) - return result, sharedArgs, nil -} - -func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns, uniqueKeyColumns *ColumnList, valueArgs, whereArgs []interface{}) (result string, sharedArgs, uniqueKeyArgs []interface{}, err error) { - if len(valueArgs) != tableColumns.Len() { - return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("value args count differs from table column count in BuildDMLUpdateQuery") - } - if len(whereArgs) != tableColumns.Len() { - return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("where args count differs from table column count in BuildDMLUpdateQuery") - } - if !sharedColumns.IsSubsetOf(tableColumns) { - return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("shared columns is not a subset of table columns in BuildDMLUpdateQuery") - } - if !uniqueKeyColumns.IsSubsetOf(sharedColumns) { - return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("unique key columns is not a subset of shared columns in BuildDMLUpdateQuery") - } - if sharedColumns.Len() == 0 { - return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("No shared columns found in BuildDMLUpdateQuery") - } - if uniqueKeyColumns.Len() == 0 { - return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("No unique key columns found in BuildDMLUpdateQuery") - } - databaseName = EscapeName(databaseName) - tableName = EscapeName(tableName) - - for _, column := range sharedColumns.Columns() { - tableOrdinal := tableColumns.Ordinals[column.Name] - arg := column.convertArg(valueArgs[tableOrdinal], false) - sharedArgs = append(sharedArgs, arg) - } - - for _, column := range uniqueKeyColumns.Columns() { - tableOrdinal := tableColumns.Ordinals[column.Name] - arg := column.convertArg(whereArgs[tableOrdinal], true) - uniqueKeyArgs = append(uniqueKeyArgs, arg) - } - - setClause, err := BuildSetPreparedClause(mappedSharedColumns) - if err != nil { - return "", sharedArgs, uniqueKeyArgs, err - } - - equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names()) - if err != nil { - return "", sharedArgs, uniqueKeyArgs, err - } - result = fmt.Sprintf(` - update /* gh-ost %s.%s */ - %s.%s - set - %s - where - %s`, - databaseName, tableName, - databaseName, tableName, - setClause, - equalsComparison, - ) - return result, sharedArgs, uniqueKeyArgs, nil -} - +// DMLDeleteQueryBuilder can build DELETE queries for DML events. +// It holds the prepared query statement so it doesn't need to be recreated every time. type DMLDeleteQueryBuilder struct { tableColumns, uniqueKeyColumns *ColumnList preparedStatement string } +// NewDMLDeleteQueryBuilder creates a new DMLDeleteQueryBuilder. +// It prepares the DELETE query statement. +// Returns an error if no unique key columns are given +// or the prepared statement cannot be built. func NewDMLDeleteQueryBuilder(databaseName, tableName string, tableColumns, uniqueKeyColumns *ColumnList) (*DMLDeleteQueryBuilder, error) { if uniqueKeyColumns.Len() == 0 { return nil, fmt.Errorf("no unique key columns found in NewDMLDeleteQueryBuilder") @@ -566,6 +443,9 @@ func NewDMLDeleteQueryBuilder(databaseName, tableName string, tableColumns, uniq return b, nil } +// BuildQuery builds the arguments array for a DML event DELETE query. +// It returns the query string and the unique key arguments array. +// Returns an error if the number of arguments is not equal to the number of table columns. func (b *DMLDeleteQueryBuilder) BuildQuery(args []interface{}) (string, []interface{}, error) { if len(args) != b.tableColumns.Len() { return "", nil, fmt.Errorf("args count differs from table column count in BuildDMLDeleteQuery") @@ -579,11 +459,17 @@ func (b *DMLDeleteQueryBuilder) BuildQuery(args []interface{}) (string, []interf return b.preparedStatement, uniqueKeyArgs, nil } +// DMLInsertQueryBuilder can build INSERT queries for DML events. +// It holds the prepared query statement so it doesn't need to be recreated every time. type DMLInsertQueryBuilder struct { tableColumns, sharedColumns *ColumnList preparedStatement string } +// NewDMLInsertQueryBuilder creates a new DMLInsertQueryBuilder. +// It prepares the INSERT query statement. +// Returns an error if no shared columns are given, the shared columns are not a subset of the table columns, +// or the prepared statement cannot be built. func NewDMLInsertQueryBuilder(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns *ColumnList) (*DMLInsertQueryBuilder, error) { if !sharedColumns.IsSubsetOf(tableColumns) { return nil, fmt.Errorf("shared columns is not a subset of table columns in NewDMLInsertQueryBuilder") @@ -619,6 +505,9 @@ func NewDMLInsertQueryBuilder(databaseName, tableName string, tableColumns, shar }, nil } +// BuildQuery builds the arguments array for a DML event INSERT query. +// It returns the query string and the shared arguments array. +// Returns an error if the number of arguments differs from the number of table columns. func (b *DMLInsertQueryBuilder) BuildQuery(args []interface{}) (string, []interface{}, error) { if len(args) != b.tableColumns.Len() { return "", nil, fmt.Errorf("args count differs from table column count in BuildDMLInsertQuery") @@ -632,11 +521,17 @@ func (b *DMLInsertQueryBuilder) BuildQuery(args []interface{}) (string, []interf return b.preparedStatement, sharedArgs, nil } +// DMLUpdateQueryBuilder can build UPDATE queries for DML events. +// It holds the prepared query statement so it doesn't need to be recreated every time. type DMLUpdateQueryBuilder struct { tableColumns, sharedColumns, uniqueKeyColumns *ColumnList preparedStatement string } +// NewDMLUpdateQueryBuilder creates a new DMLUpdateQueryBuilder. +// It prepares the UPDATE query statement. +// Returns an error if no shared columns are given, the shared columns are not a subset of the table columns, +// no unique key columns are given or the prepared statement cannot be built. func NewDMLUpdateQueryBuilder(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns, uniqueKeyColumns *ColumnList) (*DMLUpdateQueryBuilder, error) { if !sharedColumns.IsSubsetOf(tableColumns) { return nil, fmt.Errorf("shared columns is not a subset of table columns in NewDMLUpdateQueryBuilder") @@ -678,6 +573,8 @@ func NewDMLUpdateQueryBuilder(databaseName, tableName string, tableColumns, shar }, nil } +// BuildQuery builds the arguments array for a DML event UPDATE query. +// It returns the query string, the shared arguments array, and the unique key arguments array. func (b *DMLUpdateQueryBuilder) BuildQuery(valueArgs, whereArgs []interface{}) (string, []interface{}, []interface{}, error) { // TODO: move this check back to `NewDMLUpdateQueryBuilder()`, needs fix on generated columns. if !b.uniqueKeyColumns.IsSubsetOf(b.sharedColumns) { diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index fa1662138..a5cea37eb 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -393,8 +393,10 @@ func TestBuildDMLDeleteQuery(t *testing.T) { args := []interface{}{3, "testname", "first", 17, 23} { uniqueKeyColumns := NewColumnList([]string{"position"}) + builder, err := NewDMLDeleteQueryBuilder(databaseName, tableName, tableColumns, uniqueKeyColumns) + require.NoError(t, err) - query, uniqueKeyArgs, err := BuildDMLDeleteQuery(databaseName, tableName, tableColumns, uniqueKeyColumns, args) + query, uniqueKeyArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` delete /* gh-ost mydb.tbl */ @@ -408,8 +410,10 @@ func TestBuildDMLDeleteQuery(t *testing.T) { } { uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + builder, err := NewDMLDeleteQueryBuilder(databaseName, tableName, tableColumns, uniqueKeyColumns) + require.NoError(t, err) - query, uniqueKeyArgs, err := BuildDMLDeleteQuery(databaseName, tableName, tableColumns, uniqueKeyColumns, args) + query, uniqueKeyArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` delete /* gh-ost mydb.tbl */ @@ -423,8 +427,10 @@ func TestBuildDMLDeleteQuery(t *testing.T) { } { uniqueKeyColumns := NewColumnList([]string{"position", "name"}) + builder, err := NewDMLDeleteQueryBuilder(databaseName, tableName, tableColumns, uniqueKeyColumns) + require.NoError(t, err) - query, uniqueKeyArgs, err := BuildDMLDeleteQuery(databaseName, tableName, tableColumns, uniqueKeyColumns, args) + query, uniqueKeyArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` delete /* gh-ost mydb.tbl */ @@ -439,8 +445,10 @@ func TestBuildDMLDeleteQuery(t *testing.T) { { uniqueKeyColumns := NewColumnList([]string{"position", "name"}) args := []interface{}{"first", 17} + builder, err := NewDMLDeleteQueryBuilder(databaseName, tableName, tableColumns, uniqueKeyColumns) + require.NoError(t, err) - _, _, err := BuildDMLDeleteQuery(databaseName, tableName, tableColumns, uniqueKeyColumns, args) + _, _, err = builder.BuildQuery(args) require.Error(t, err) } } @@ -450,10 +458,12 @@ func TestBuildDMLDeleteQuerySignedUnsigned(t *testing.T) { tableName := "tbl" tableColumns := NewColumnList([]string{"id", "name", "rank", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"position"}) + builder, err := NewDMLDeleteQueryBuilder(databaseName, tableName, tableColumns, uniqueKeyColumns) + require.NoError(t, err) { // test signed (expect no change) args := []interface{}{3, "testname", "first", -1, 23} - query, uniqueKeyArgs, err := BuildDMLDeleteQuery(databaseName, tableName, tableColumns, uniqueKeyColumns, args) + query, uniqueKeyArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` delete /* gh-ost mydb.tbl */ @@ -469,7 +479,7 @@ func TestBuildDMLDeleteQuerySignedUnsigned(t *testing.T) { // test unsigned args := []interface{}{3, "testname", "first", int8(-1), 23} uniqueKeyColumns.SetUnsigned("position") - query, uniqueKeyArgs, err := BuildDMLDeleteQuery(databaseName, tableName, tableColumns, uniqueKeyColumns, args) + query, uniqueKeyArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` delete /* gh-ost mydb.tbl */ @@ -490,7 +500,9 @@ func TestBuildDMLInsertQuery(t *testing.T) { args := []interface{}{3, "testname", "first", 17, 23} { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) + builder, err := NewDMLInsertQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns) + require.NoError(t, err) + query, sharedArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` replace /* gh-ost mydb.tbl */ @@ -504,7 +516,9 @@ func TestBuildDMLInsertQuery(t *testing.T) { } { sharedColumns := NewColumnList([]string{"position", "name", "age", "id"}) - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) + builder, err := NewDMLInsertQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns) + require.NoError(t, err) + query, sharedArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` replace /* gh-ost mydb.tbl */ @@ -518,12 +532,12 @@ func TestBuildDMLInsertQuery(t *testing.T) { } { sharedColumns := NewColumnList([]string{"position", "name", "surprise", "id"}) - _, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) + _, err := NewDMLInsertQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns) require.Error(t, err) } { sharedColumns := NewColumnList([]string{}) - _, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) + _, err := NewDMLInsertQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns) require.Error(t, err) } } @@ -537,7 +551,9 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) { // testing signed args := []interface{}{3, "testname", "first", int8(-1), 23} sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) + builder, err := NewDMLInsertQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns) + require.NoError(t, err) + query, sharedArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` replace /* gh-ost mydb.tbl */ @@ -553,7 +569,9 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) { // testing unsigned args := []interface{}{3, "testname", "first", int8(-1), 23} sharedColumns.SetUnsigned("position") - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) + builder, err := NewDMLInsertQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns) + require.NoError(t, err) + query, sharedArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` replace /* gh-ost mydb.tbl */ @@ -569,7 +587,9 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) { // testing unsigned args := []interface{}{3, "testname", "first", int32(-1), 23} sharedColumns.SetUnsigned("position") - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) + builder, err := NewDMLInsertQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns) + require.NoError(t, err) + query, sharedArgs, err := builder.BuildQuery(args) require.NoError(t, err) expected := ` replace /* gh-ost mydb.tbl */ @@ -592,7 +612,9 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"position"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + builder, err := NewDMLUpdateQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns) + require.NoError(t, err) + query, sharedArgs, uniqueKeyArgs, err := builder.BuildQuery(valueArgs, whereArgs) require.NoError(t, err) expected := ` update /* gh-ost mydb.tbl */ @@ -608,7 +630,9 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"position", "name"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + builder, err := NewDMLUpdateQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns) + require.NoError(t, err) + query, sharedArgs, uniqueKeyArgs, err := builder.BuildQuery(valueArgs, whereArgs) require.NoError(t, err) expected := ` update /* gh-ost mydb.tbl */ @@ -624,7 +648,9 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"age"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + builder, err := NewDMLUpdateQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns) + require.NoError(t, err) + query, sharedArgs, uniqueKeyArgs, err := builder.BuildQuery(valueArgs, whereArgs) require.NoError(t, err) expected := ` update /* gh-ost mydb.tbl */ @@ -640,7 +666,9 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"age", "position", "id", "name"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + builder, err := NewDMLUpdateQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns) + require.NoError(t, err) + query, sharedArgs, uniqueKeyArgs, err := builder.BuildQuery(valueArgs, whereArgs) require.NoError(t, err) expected := ` update /* gh-ost mydb.tbl */ @@ -656,20 +684,24 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"age", "surprise"}) - _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + builder, err := NewDMLUpdateQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns) + require.NoError(t, err) + _, _, _, err = builder.BuildQuery(valueArgs, whereArgs) require.Error(t, err) } { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{}) - _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + _, err := NewDMLUpdateQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns) require.Error(t, err) } { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) mappedColumns := NewColumnList([]string{"id", "name", "role", "age"}) uniqueKeyColumns := NewColumnList([]string{"id"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, mappedColumns, uniqueKeyColumns, valueArgs, whereArgs) + builder, err := NewDMLUpdateQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, mappedColumns, uniqueKeyColumns) + require.NoError(t, err) + query, sharedArgs, uniqueKeyArgs, err := builder.BuildQuery(valueArgs, whereArgs) require.NoError(t, err) expected := ` update /* gh-ost mydb.tbl */ @@ -692,9 +724,11 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { whereArgs := []interface{}{3, "testname", "findme", int8(-3), 56} sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"position"}) + builder, err := NewDMLUpdateQueryBuilder(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns) + require.NoError(t, err) { // test signed - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := builder.BuildQuery(valueArgs, whereArgs) require.NoError(t, err) expected := ` update /* gh-ost mydb.tbl */ @@ -711,7 +745,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { // test unsigned sharedColumns.SetUnsigned("age") uniqueKeyColumns.SetUnsigned("position") - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := builder.BuildQuery(valueArgs, whereArgs) require.NoError(t, err) expected := ` update /* gh-ost mydb.tbl */