VDiff: Save lastpk value for source and target (#17493)

Signed-off-by: Matt Lord <[email protected]>

go/test/endtoend/vreplication/config_test.go

@@ -45,12 +45,18 @@ import (
 // default collation as it has to work across versions and the 8.0 default does not exist in 5.7.
 var (
 	// All standard user tables should have a primary key and at least one secondary key.
-	customerTypes        = []string{"'individual'", "'soho'", "'enterprise'"}
+	customerTypes         = []string{"'individual'", "'soho'", "'enterprise'"}
+	customerTableTemplate = `create table customer(cid int auto_increment, name varchar(128) collate utf8mb4_bin, meta json default null,
+  industryCategory varchar(100) generated always as (json_extract(meta, _utf8mb4'$.industry')) virtual, typ enum(%s),
+  sport set('football','cricket','baseball'), ts timestamp not null default current_timestamp, bits bit(2) default b'11', date1 datetime not null default '0000-00-00 00:00:00',
+	date2 datetime not null default '2021-00-01 00:00:00', dec80 decimal(8,0), blb blob, primary key(%s), key(name)) CHARSET=utf8mb4`
+	customerTable = fmt.Sprintf(customerTableTemplate, strings.Join(customerTypes, ","), "cid,typ" /* PK columns */)
+	// customerTableModifiedPK has a PK on (cid) vs (cid,typ).
+	customerTableModifiedPK = fmt.Sprintf(customerTableTemplate, strings.Join(customerTypes, ","), "cid" /* PK columns */)
+
 	initialProductSchema = fmt.Sprintf(`
 create table product(pid int, description varbinary(128), date1 datetime not null default '0000-00-00 00:00:00', date2 datetime not null default '2021-00-01 00:00:00', primary key(pid), key(date1,date2)) CHARSET=utf8mb4;
-create table customer(cid int auto_increment, name varchar(128) collate utf8mb4_bin, meta json default null, industryCategory varchar(100) generated always as (json_extract(meta, _utf8mb4'$.industry')) virtual,
-  typ enum(%s), sport set('football','cricket','baseball'), ts timestamp not null default current_timestamp, bits bit(2) default b'11', date1 datetime not null default '0000-00-00 00:00:00',
-	date2 datetime not null default '2021-00-01 00:00:00', dec80 decimal(8,0), blb blob, primary key(cid,typ), key(name)) CHARSET=utf8mb4;
+%s;
 create table customer_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';
 create table merchant(mname varchar(128), category varchar(128), primary key(mname), key(category)) CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
 create table orders(oid int, cid int, pid int, mname varchar(128), price int, qty int, total int as (qty * price), total2 int as (qty * price) stored, primary key(oid), key(pid), key(cid)) CHARSET=utf8;
@@ -69,7 +75,8 @@ create table  `+"`blüb_tbl`"+` (id int, val1 varchar(20), `+"`blöb1`"+` blob,
 create table reftable (id int, val1 varchar(20), primary key(id), key(val1));
 create table loadtest (id int, name varchar(256), primary key(id), key(name));
 create table nopk (name varchar(128), age int unsigned);
-`, strings.Join(customerTypes, ","))
+`, customerTable)
+
 	// These should always be ignored in vreplication
 	internalSchema = `
  create table _1e275eef_3b20_11eb_a38f_04ed332e05c2_20201210204529_gho(id int, val varbinary(128), primary key(id));

go/test/endtoend/vreplication/multi_tenant_test.go

@@ -91,7 +91,15 @@ type multiTenantMigration struct {
 }
 
 const (
-	mtSchema  = "create table t1(id int, tenant_id int, primary key(id, tenant_id)) Engine=InnoDB"
+	// The source/mt schema does not have the tenant_id column in the PK as adding a
+	// column to a table can be done as an INSTANT operation whereas modifying a table's
+	// PK requires a full table rebuild. So as a practical matter in production the
+	// source schema will likely have the tenant_id column, but NOT have it be part of
+	// the PK.
+	mtSchema = "create table t1(id int, tenant_id int, primary key(id)) Engine=InnoDB"
+	// The target/st schema must have the tenant_id column in the PK and the primary
+	// vindex.
+	stSchema  = "create table t1(id int, tenant_id int, primary key(id, tenant_id)) Engine=InnoDB"
 	mtVSchema = `
 {
   "multi_tenant_spec": {
@@ -127,7 +135,6 @@ const (
   }
 }
 `
-	stSchema  = mtSchema
 	stVSchema = `
 {
   "tables": {
@@ -429,8 +436,11 @@ func (mtm *multiTenantMigration) insertSomeData(t *testing.T, tenantId int64, ke
 	defer closeConn()
 	idx := mtm.getLastID(tenantId)
 	for i := idx + 1; i <= idx+numRows; i++ {
+		// The source table has a PK on id only, so we have to make the id value
+		// unique rather than relying on the combination of (id, tenant_id) for
+		// our uniqueness.
 		execQueryWithRetry(t, vtgateConn,
-			fmt.Sprintf("insert into %s.t1(id, tenant_id) values(%d, %d)", keyspace, i, tenantId), queryTimeout)
+			fmt.Sprintf("insert into %s.t1(id, tenant_id) values(%d, %d)", keyspace, i+(tenantId*1e4), tenantId), queryTimeout)
 	}
 	mtm.setLastID(tenantId, idx+numRows)
 }

go/test/endtoend/vreplication/vdiff2_test.go

@@ -175,6 +175,15 @@ func TestVDiff2(t *testing.T) {
 	require.NoError(t, err)
 	verifyClusterHealth(t, vc)
 
+	// Pre-create the customer table on the target keyspace, with the primary key on
+	// (cid) vs (cid,typ) on the source. This confirms that we are able to properly
+	// diff the table when the source and target have a different PK definition.
+	// Remove the 0 date restrictions as the customer table uses them in its DEFAULTs.
+	execVtgateQuery(t, vtgateConn, targetKs, "set @@session.sql_mode='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'")
+	execVtgateQuery(t, vtgateConn, targetKs, customerTableModifiedPK)
+	// Set the sql_mode back to the default.
+	execVtgateQuery(t, vtgateConn, targetKs, "set @@session.sql_mode='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'")
+
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			// Primary tablets for any new shards are added in the first cell.