Skip to content

Commit e08d151

Browse files
mcrauwelrohit-nayak-ps
mcrauwel
and
rohit-nayak-ps
authored
implement --max-report-sample-rows for VDiff (#14437)
Signed-off-by: Matthias Crauwels <[email protected]> Signed-off-by: Rohit Nayak <[email protected]> Co-authored-by: Rohit Nayak <[email protected]>
1 parent 8b7ed8d commit e08d151

File tree

15 files changed

+766
-324
lines changed

15 files changed

+766
-324
lines changed

‎go/cmd/vtctldclient/command/vreplication/vdiff/vdiff.go

+3
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ var (
6060
Limit uint32 // We only accept positive values but pass on an int64
6161
FilteredReplicationWaitTime time.Duration
6262
DebugQuery bool
63+
MaxReportSampleRows uint32 // We only accept positive values but pass on an int64
6364
OnlyPKs bool
6465
UpdateTableStats bool
6566
MaxExtraRowsToCompare uint32 // We only accept positive values but pass on an int64
@@ -279,6 +280,7 @@ func commandCreate(cmd *cobra.Command, args []string) error {
279280
Wait: createOptions.Wait,
280281
WaitUpdateInterval: protoutil.DurationToProto(createOptions.WaitUpdateInterval),
281282
AutoRetry: createOptions.AutoRetry,
283+
MaxReportSampleRows: int64(createOptions.MaxReportSampleRows),
282284
})
283285

284286
if err != nil {
@@ -863,6 +865,7 @@ func registerCommands(root *cobra.Command) {
863865
create.Flags().DurationVar(&createOptions.FilteredReplicationWaitTime, "filtered-replication-wait-time", 30*time.Second, "Specifies the maximum time to wait, in seconds, for replication to catch up when syncing tablet streams.")
864866
create.Flags().Uint32Var(&createOptions.Limit, "limit", math.MaxUint32, "Max rows to stop comparing after.")
865867
create.Flags().BoolVar(&createOptions.DebugQuery, "debug-query", false, "Adds a mysql query to the report that can be used for further debugging.")
868+
create.Flags().Uint32Var(&createOptions.MaxReportSampleRows, "max-report-sample-rows", 10, "Maximum number of row differences to report (0 for all differences). NOTE: when increasing this value it is highly recommended to also specify --only-pks")
866869
create.Flags().BoolVar(&createOptions.OnlyPKs, "only-pks", false, "When reporting missing rows, only show primary keys in the report.")
867870
create.Flags().StringSliceVar(&createOptions.Tables, "tables", nil, "Only run vdiff for these tables in the workflow.")
868871
create.Flags().Uint32Var(&createOptions.MaxExtraRowsToCompare, "max-extra-rows-to-compare", 1000, "If there are collation differences between the source and target, you can have rows that are identical but simply returned in a different order from MySQL. We will do a second pass to compare the rows for any actual differences in this case and this flag allows you to control the resources used for this operation.")

‎go/cmd/vtctldclient/command/vreplication/vdiff/vdiff_test.go

+271
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,277 @@ func TestVDiffUnsharded(t *testing.T) {
269269
}
270270
}
271271
]`),
272+
}, {
273+
id: "9", // --max-vdiff-report-rows=20 --only-pks
274+
result: sqltypes.MakeTestResult(fields,
275+
"completed||t1|"+UUID+"|completed|30|"+starttime+"|30|"+comptime+"|1|"+
276+
`{"TableName": "t1", "MatchingRows": 10, "ProcessedRows": 30, "MismatchedRows": 20, "ExtraRowsSource": 0, `+
277+
`"ExtraRowsTarget": 0, "MismatchedRowsSample": [`+
278+
`{"Source": {"Row": {"c1": "2"}}, "Target": {"Row": {"c1": "2"}}},`+
279+
`{"Source": {"Row": {"c1": "3"}}, "Target": {"Row": {"c1": "3"}}},`+
280+
`{"Source": {"Row": {"c1": "4"}}, "Target": {"Row": {"c1": "4"}}},`+
281+
`{"Source": {"Row": {"c1": "5"}}, "Target": {"Row": {"c1": "5"}}},`+
282+
`{"Source": {"Row": {"c1": "6"}}, "Target": {"Row": {"c1": "6"}}},`+
283+
`{"Source": {"Row": {"c1": "7"}}, "Target": {"Row": {"c1": "7"}}},`+
284+
`{"Source": {"Row": {"c1": "8"}}, "Target": {"Row": {"c1": "8"}}},`+
285+
`{"Source": {"Row": {"c1": "9"}}, "Target": {"Row": {"c1": "9"}}},`+
286+
`{"Source": {"Row": {"c1": "10"}}, "Target": {"Row": {"c1": "10"}}},`+
287+
`{"Source": {"Row": {"c1": "11"}}, "Target": {"Row": {"c1": "11"}}},`+
288+
`{"Source": {"Row": {"c1": "12"}}, "Target": {"Row": {"c1": "12"}}},`+
289+
`{"Source": {"Row": {"c1": "13"}}, "Target": {"Row": {"c1": "13"}}},`+
290+
`{"Source": {"Row": {"c1": "14"}}, "Target": {"Row": {"c1": "14"}}},`+
291+
`{"Source": {"Row": {"c1": "15"}}, "Target": {"Row": {"c1": "15"}}},`+
292+
`{"Source": {"Row": {"c1": "16"}}, "Target": {"Row": {"c1": "16"}}},`+
293+
`{"Source": {"Row": {"c1": "17"}}, "Target": {"Row": {"c1": "17"}}},`+
294+
`{"Source": {"Row": {"c1": "18"}}, "Target": {"Row": {"c1": "18"}}},`+
295+
`{"Source": {"Row": {"c1": "19"}}, "Target": {"Row": {"c1": "19"}}},`+
296+
`{"Source": {"Row": {"c1": "20"}}, "Target": {"Row": {"c1": "20"}}},`+
297+
`{"Source": {"Row": {"c1": "21"}}, "Target": {"Row": {"c1": "21"}}}`+
298+
`]}`),
299+
report: fmt.Sprintf(badReportfmt,
300+
env.targetKeyspace, UUID, 30, true, starttime, comptime, 30, 10, 20, 0, 0, 30, 10, 20, 0, 0,
301+
`"MismatchedRowsSample": [
302+
{
303+
"Source": {
304+
"Row": {
305+
"c1": "2"
306+
}
307+
},
308+
"Target": {
309+
"Row": {
310+
"c1": "2"
311+
}
312+
}
313+
},
314+
{
315+
"Source": {
316+
"Row": {
317+
"c1": "3"
318+
}
319+
},
320+
"Target": {
321+
"Row": {
322+
"c1": "3"
323+
}
324+
}
325+
},
326+
{
327+
"Source": {
328+
"Row": {
329+
"c1": "4"
330+
}
331+
},
332+
"Target": {
333+
"Row": {
334+
"c1": "4"
335+
}
336+
}
337+
},
338+
{
339+
"Source": {
340+
"Row": {
341+
"c1": "5"
342+
}
343+
},
344+
"Target": {
345+
"Row": {
346+
"c1": "5"
347+
}
348+
}
349+
},
350+
{
351+
"Source": {
352+
"Row": {
353+
"c1": "6"
354+
}
355+
},
356+
"Target": {
357+
"Row": {
358+
"c1": "6"
359+
}
360+
}
361+
},
362+
{
363+
"Source": {
364+
"Row": {
365+
"c1": "7"
366+
}
367+
},
368+
"Target": {
369+
"Row": {
370+
"c1": "7"
371+
}
372+
}
373+
},
374+
{
375+
"Source": {
376+
"Row": {
377+
"c1": "8"
378+
}
379+
},
380+
"Target": {
381+
"Row": {
382+
"c1": "8"
383+
}
384+
}
385+
},
386+
{
387+
"Source": {
388+
"Row": {
389+
"c1": "9"
390+
}
391+
},
392+
"Target": {
393+
"Row": {
394+
"c1": "9"
395+
}
396+
}
397+
},
398+
{
399+
"Source": {
400+
"Row": {
401+
"c1": "10"
402+
}
403+
},
404+
"Target": {
405+
"Row": {
406+
"c1": "10"
407+
}
408+
}
409+
},
410+
{
411+
"Source": {
412+
"Row": {
413+
"c1": "11"
414+
}
415+
},
416+
"Target": {
417+
"Row": {
418+
"c1": "11"
419+
}
420+
}
421+
},
422+
{
423+
"Source": {
424+
"Row": {
425+
"c1": "12"
426+
}
427+
},
428+
"Target": {
429+
"Row": {
430+
"c1": "12"
431+
}
432+
}
433+
},
434+
{
435+
"Source": {
436+
"Row": {
437+
"c1": "13"
438+
}
439+
},
440+
"Target": {
441+
"Row": {
442+
"c1": "13"
443+
}
444+
}
445+
},
446+
{
447+
"Source": {
448+
"Row": {
449+
"c1": "14"
450+
}
451+
},
452+
"Target": {
453+
"Row": {
454+
"c1": "14"
455+
}
456+
}
457+
},
458+
{
459+
"Source": {
460+
"Row": {
461+
"c1": "15"
462+
}
463+
},
464+
"Target": {
465+
"Row": {
466+
"c1": "15"
467+
}
468+
}
469+
},
470+
{
471+
"Source": {
472+
"Row": {
473+
"c1": "16"
474+
}
475+
},
476+
"Target": {
477+
"Row": {
478+
"c1": "16"
479+
}
480+
}
481+
},
482+
{
483+
"Source": {
484+
"Row": {
485+
"c1": "17"
486+
}
487+
},
488+
"Target": {
489+
"Row": {
490+
"c1": "17"
491+
}
492+
}
493+
},
494+
{
495+
"Source": {
496+
"Row": {
497+
"c1": "18"
498+
}
499+
},
500+
"Target": {
501+
"Row": {
502+
"c1": "18"
503+
}
504+
}
505+
},
506+
{
507+
"Source": {
508+
"Row": {
509+
"c1": "19"
510+
}
511+
},
512+
"Target": {
513+
"Row": {
514+
"c1": "19"
515+
}
516+
}
517+
},
518+
{
519+
"Source": {
520+
"Row": {
521+
"c1": "20"
522+
}
523+
},
524+
"Target": {
525+
"Row": {
526+
"c1": "20"
527+
}
528+
}
529+
},
530+
{
531+
"Source": {
532+
"Row": {
533+
"c1": "21"
534+
}
535+
},
536+
"Target": {
537+
"Row": {
538+
"c1": "21"
539+
}
540+
}
541+
}
542+
]`),
272543
},
273544
}
274545

0 commit comments

Comments
 (0)