From 57d918ba0d3d28e9a58200495c28004d3bb4d851 Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Wed, 31 Jul 2024 14:18:13 -0400 Subject: [PATCH 1/2] kvserver: split snapshot SSTables for mvcc keys into multiple SSTs Previously, we'd only create one sstable for all mvcc keys in a range when ingesting a rebalance/recovery snapshot into Pebble. This increased write-amp in Pebble as more sstables would have to be compacted into it (or the sstable then split into smaller ones in Pebble), and had other consequences such as massive filter blocks in the large singular sstable. This change adds a new cluster setting, kv.snapshot_rebalance.max_sst_size, that sets the max size of the sstables containing user/mvcc keys in a range. If an sstable exceeds this size in multiSSTWriter, we roll over that sstable and create a new one. Epic: CRDB-8471 Fixes: #67284 Release note (performance improvement): Reduce the write-amplification impact of rebalances by splitting snapshot sstable files into smaller ones before ingesting them into Pebble. --- pkg/kv/kvserver/BUILD.bazel | 1 + .../replica_sst_snapshot_storage_test.go | 127 ++++++- pkg/kv/kvserver/store_snapshot.go | 314 ++++++++++++++---- pkg/storage/engine_key.go | 5 + pkg/storage/sst_writer.go | 11 + 5 files changed, 391 insertions(+), 67 deletions(-) diff --git a/pkg/kv/kvserver/BUILD.bazel b/pkg/kv/kvserver/BUILD.bazel index 703b5af4d4b..422148e61cd 100644 --- a/pkg/kv/kvserver/BUILD.bazel +++ b/pkg/kv/kvserver/BUILD.bazel @@ -523,6 +523,7 @@ go_test( "@com_github_cockroachdb_errors//oserror", "@com_github_cockroachdb_logtags//:logtags", "@com_github_cockroachdb_pebble//:pebble", + "@com_github_cockroachdb_pebble//sstable", "@com_github_cockroachdb_pebble//vfs", "@com_github_cockroachdb_redact//:redact", "@com_github_dustin_go_humanize//:go-humanize", diff --git a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go index 0ded1aa2fc5..6e13695a7e9 100644 --- a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go +++ b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go @@ -31,6 +31,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" "github.com/cockroachdb/errors/oserror" + "github.com/cockroachdb/pebble/sstable" + "github.com/cockroachdb/pebble/vfs" "github.com/stretchr/testify/require" "golang.org/x/time/rate" ) @@ -276,10 +278,12 @@ func TestMultiSSTWriterInitSST(t *testing.T) { EndKey: roachpb.RKeyMax, } keySpans := rditer.MakeReplicatedKeySpans(&desc) + localSpans := keySpans[:len(keySpans)-1] + mvccSpan := keySpans[len(keySpans)-1] msstw, err := newMultiSSTWriter( - ctx, cluster.MakeTestingClusterSettings(), scratch, keySpans, 0, - false, /* skipRangeDelForLastSpan */ + ctx, cluster.MakeTestingClusterSettings(), scratch, localSpans, mvccSpan, 0, + false, /* skipRangeDelForMVCCSpan */ ) require.NoError(t, err) _, err = msstw.Finish(ctx) @@ -315,6 +319,117 @@ func TestMultiSSTWriterInitSST(t *testing.T) { } } +func buildIterForScratch( + t *testing.T, keySpans []roachpb.Span, scratch *SSTSnapshotStorageScratch, +) (storage.MVCCIterator, error) { + var openFiles []sstable.ReadableFile + for _, sstPath := range scratch.SSTs()[len(keySpans)-1:] { + f, err := vfs.Default.Open(sstPath) + require.NoError(t, err) + openFiles = append(openFiles, f) + } + mvccSpan := keySpans[len(keySpans)-1] + + return storage.NewSSTIterator([][]sstable.ReadableFile{openFiles}, storage.IterOptions{ + LowerBound: mvccSpan.Key, + UpperBound: mvccSpan.EndKey, + }) +} + +// TestMultiSSTWriterSize tests the effect of lowering the max size +// of sstables in a multiSSTWriter, and ensuring that the produced sstables +// are still correct. +func TestMultiSSTWriterSize(t *testing.T) { + defer leaktest.AfterTest(t)() + + ctx := context.Background() + testRangeID := roachpb.RangeID(1) + testSnapUUID := uuid.Must(uuid.FromBytes([]byte("foobar1234567890"))) + testLimiter := rate.NewLimiter(rate.Inf, 0) + + cleanup, eng := newOnDiskEngine(ctx, t) + defer cleanup() + defer eng.Close() + + sstSnapshotStorage := NewSSTSnapshotStorage(eng, testLimiter) + ref := sstSnapshotStorage.NewScratchSpace(testRangeID, testSnapUUID) + scratch := sstSnapshotStorage.NewScratchSpace(testRangeID, testSnapUUID) + settings := cluster.MakeTestingClusterSettings() + + desc := roachpb.RangeDescriptor{ + StartKey: roachpb.RKey("d"), + EndKey: roachpb.RKeyMax, + } + keySpans := rditer.MakeReplicatedKeySpans(&desc) + localSpans := keySpans[:len(keySpans)-1] + mvccSpan := keySpans[len(keySpans)-1] + + // Make a reference msstw with the default size. + referenceMsstw, err := newMultiSSTWriter(ctx, settings, ref, localSpans, mvccSpan, 0, false) + require.NoError(t, err) + require.Equal(t, int64(0), referenceMsstw.dataSize) + + for i := range localSpans { + require.NoError(t, referenceMsstw.Put(ctx, storage.EngineKey{Key: localSpans[i].Key}, []byte("foo"))) + } + + for i := 0; i < 100; i++ { + require.NoError(t, referenceMsstw.Put(ctx, storage.EngineKey{Key: roachpb.Key(append(desc.StartKey, byte(i)))}, []byte("foobarbaz"))) + } + _, err = referenceMsstw.Finish(ctx) + require.NoError(t, err) + + refIter, err := buildIterForScratch(t, keySpans, ref) + require.NoError(t, err) + defer refIter.Close() + + MaxSnapshotSSTableSize.Override(ctx, &settings.SV, 100) + + multiSSTWriter, err := newMultiSSTWriter(ctx, settings, scratch, localSpans, mvccSpan, 0, false) + require.NoError(t, err) + require.Equal(t, int64(0), multiSSTWriter.dataSize) + + for i := range localSpans { + require.NoError(t, multiSSTWriter.Put(ctx, storage.EngineKey{Key: localSpans[i].Key}, []byte("foo"))) + } + + for i := 0; i < 100; i++ { + require.NoError(t, multiSSTWriter.Put(ctx, storage.EngineKey{Key: roachpb.Key(append(desc.StartKey, byte(i)))}, []byte("foobarbaz"))) + } + + _, err = multiSSTWriter.Finish(ctx) + require.NoError(t, err) + require.Greater(t, len(scratch.SSTs()), len(ref.SSTs())) + + iter, err := buildIterForScratch(t, keySpans, scratch) + require.NoError(t, err) + defer iter.Close() + + iter.SeekGE(storage.MVCCKey{Key: mvccSpan.Key}) + refIter.SeekGE(storage.MVCCKey{Key: mvccSpan.Key}) + valid, err := iter.Valid() + valid2, err2 := refIter.Valid() + require.NoError(t, err) + require.NoError(t, err2) + + for valid && valid2 { + + require.Equal(t, iter.UnsafeKey(), refIter.UnsafeKey()) + val, err := iter.UnsafeValue() + require.NoError(t, err) + val2, err2 := refIter.UnsafeValue() + require.NoError(t, err2) + require.Equal(t, val, val2) + iter.Next() + refIter.Next() + valid, err = iter.Valid() + valid2, err2 = refIter.Valid() + require.NoError(t, err) + require.NoError(t, err2) + } + require.Equal(t, valid, valid2) +} + // TestMultiSSTWriterAddLastSpan tests that multiSSTWriter initializes each of // the SST files associated with the replicated key ranges by writing a range // deletion tombstone that spans the entire range of each respectively, except @@ -342,14 +457,16 @@ func TestMultiSSTWriterAddLastSpan(t *testing.T) { EndKey: roachpb.RKeyMax, } keySpans := rditer.MakeReplicatedKeySpans(&desc) + localSpans := keySpans[:len(keySpans)-1] + mvccSpan := keySpans[len(keySpans)-1] msstw, err := newMultiSSTWriter( - ctx, cluster.MakeTestingClusterSettings(), scratch, keySpans, 0, - true, /* skipRangeDelForLastSpan */ + ctx, cluster.MakeTestingClusterSettings(), scratch, localSpans, mvccSpan, 0, + true, /* skipRangeDelForMVCCSpan */ ) require.NoError(t, err) if addRangeDel { - require.NoError(t, msstw.addRangeDelForLastSpan()) + require.NoError(t, msstw.addClearForMVCCSpan()) } testKey := storage.MVCCKey{Key: roachpb.RKey("d1").AsRawKey(), Timestamp: hlc.Timestamp{WallTime: 1}} testEngineKey, _ := storage.DecodeEngineKey(storage.EncodeMVCCKey(testKey)) diff --git a/pkg/kv/kvserver/store_snapshot.go b/pkg/kv/kvserver/store_snapshot.go index 6fc736d0d3e..3e5cfed4091 100644 --- a/pkg/kv/kvserver/store_snapshot.go +++ b/pkg/kv/kvserver/store_snapshot.go @@ -11,7 +11,9 @@ package kvserver import ( + "bytes" "context" + "fmt" "io" "time" @@ -63,6 +65,16 @@ const ( tagSnapshotTiming = "snapshot_timing_tag" ) +// MaxSnapshotSSTableSize is the maximum size of an sstable containing MVCC/user keys +// in a snapshot before we truncate and write a new snapshot sstable. +var MaxSnapshotSSTableSize = settings.RegisterByteSizeSetting( + settings.SystemOnly, + "kv.snapshot_rebalance.max_sst_size", + "maximum size of a rebalance or recovery SST size", + 128<<20, // 128 MB + settings.PositiveInt, +) + // snapshotMetrics contains metrics on the number and size of snapshots in // progress or in the snapshot queue. type snapshotMetrics struct { @@ -116,10 +128,23 @@ type kvBatchSnapshotStrategy struct { // multiSSTWriter is a wrapper around an SSTWriter and SSTSnapshotStorageScratch // that handles chunking SSTs and persisting them to disk. type multiSSTWriter struct { - st *cluster.Settings - scratch *SSTSnapshotStorageScratch - currSST storage.SSTWriter - keySpans []roachpb.Span + st *cluster.Settings + scratch *SSTSnapshotStorageScratch + currSST storage.SSTWriter + // localKeySpans are key spans that are considered unsplittable across sstables, and + // represent the range's range local key spans. In contrast, mvccKeySpan can be split + // across multiple sstables if one of them exceeds maxSSTSize. The expectation is + // that for large ranges, keys in mvccKeySpan will dominate in size compared to keys + // in localKeySpans. + localKeySpans []roachpb.Span + mvccKeySpan roachpb.Span + // mvccSSTSpans reflects the actual split of the mvccKeySpan into constituent + // sstables. + mvccSSTSpans []storage.EngineKeyRange + // currSpan is the index of the current span being written to. The first + // len(localKeySpans) spans are localKeySpans, and the rest are mvccSSTSpans. + // In a sense, currSpan indexes into a slice composed of + // append(localKeySpans, mvccSSTSpans). currSpan int // The approximate size of the SST chunk to buffer in memory on the receiver // before flushing to disk. @@ -129,33 +154,84 @@ type multiSSTWriter struct { dataSize int64 // The total size of the SSTs. sstSize int64 - // if skipRangeDelForLastSpan is true, the last span is not ClearRanged in the - // same sstable. We rely on the caller to take care of clearing this span - // through a different process (eg. IngestAndExcise on pebble). - skipRangeDelForLastSpan bool + // if skipClearForMVCCSpan is true, the MVCC span is not ClearEngineRange()d in + // the same sstable. We rely on the caller to take care of clearing this span + // through a different process (eg. IngestAndExcise on pebble). Note that + // having this bool to true also disables all range key fragmentation + // and splitting of sstables in the mvcc span. + skipClearForMVCCSpan bool + // maxSSTSize is the maximum size to use for SSTs containing MVCC/user keys. + // Once the sstable writer reaches this size, it will be finalized and a new + // sstable will be created. + maxSSTSize int64 + // rangeKeyFrag is used to fragment range keys across the mvcc key spans. + rangeKeyFrag rangekey.Fragmenter } func newMultiSSTWriter( ctx context.Context, st *cluster.Settings, scratch *SSTSnapshotStorageScratch, - keySpans []roachpb.Span, + localKeySpans []roachpb.Span, + mvccKeySpan roachpb.Span, sstChunkSize int64, - skipRangeDelForLastSpan bool, -) (multiSSTWriter, error) { - msstw := multiSSTWriter{ - st: st, - scratch: scratch, - keySpans: keySpans, - sstChunkSize: sstChunkSize, - skipRangeDelForLastSpan: skipRangeDelForLastSpan, + skipClearForMVCCSpan bool, +) (*multiSSTWriter, error) { + msstw := &multiSSTWriter{ + st: st, + scratch: scratch, + localKeySpans: localKeySpans, + mvccKeySpan: mvccKeySpan, + mvccSSTSpans: []storage.EngineKeyRange{{ + Start: storage.EngineKey{Key: mvccKeySpan.Key}, + End: storage.EngineKey{Key: mvccKeySpan.EndKey}, + }}, + sstChunkSize: sstChunkSize, + skipClearForMVCCSpan: skipClearForMVCCSpan, + } + if !skipClearForMVCCSpan { + // If skipClearForMVCCSpan is true, we don't split the MVCC span across + // multiple sstables, as addClearForMVCCSpan could be called by the caller + // at any time. + msstw.maxSSTSize = MaxSnapshotSSTableSize.Get(&st.SV) + } + msstw.rangeKeyFrag = rangekey.Fragmenter{ + Cmp: storage.EngineComparer.Compare, + Format: storage.EngineComparer.FormatKey, + Emit: msstw.emitRangeKey, } + if err := msstw.initSST(ctx); err != nil { return msstw, err } return msstw, nil } +func (msstw *multiSSTWriter) emitRangeKey(key rangekey.Span) { + for i := range key.Keys { + if err := msstw.currSST.PutInternalRangeKey(key.Start, key.End, key.Keys[i]); err != nil { + panic(fmt.Sprintf("failed to put range key in sst: %s", err)) + } + } +} + +// currentSpan returns the current user-provided span that +// is being written to. Note that this does not account for +// mvcc keys being split across multiple sstables. +func (msstw *multiSSTWriter) currentSpan() roachpb.Span { + if msstw.currSpanIsMVCCSpan() { + return msstw.mvccKeySpan + } + return msstw.localKeySpans[msstw.currSpan] +} + +func (msstw *multiSSTWriter) currSpanIsMVCCSpan() bool { + if msstw.currSpan >= len(msstw.localKeySpans)+len(msstw.mvccSSTSpans) { + panic("current span is out of bounds") + } + return msstw.currSpan >= len(msstw.localKeySpans) +} + func (msstw *multiSSTWriter) initSST(ctx context.Context) error { newSSTFile, err := msstw.scratch.NewFile(ctx, msstw.sstChunkSize) if err != nil { @@ -163,26 +239,94 @@ func (msstw *multiSSTWriter) initSST(ctx context.Context) error { } newSST := storage.MakeIngestionSSTWriter(ctx, msstw.st, newSSTFile) msstw.currSST = newSST - if msstw.skipRangeDelForLastSpan && msstw.currSpan == len(msstw.keySpans)-1 { - // Skip this ClearRange, as it will be excised at ingestion time in the - // engine instead. - return nil - } - if err := msstw.currSST.ClearRawRange( - msstw.keySpans[msstw.currSpan].Key, msstw.keySpans[msstw.currSpan].EndKey, - true /* pointKeys */, true, /* rangeKeys */ - ); err != nil { - msstw.currSST.Close() - return errors.Wrap(err, "failed to clear range on sst file writer") + if !msstw.currSpanIsMVCCSpan() || (!msstw.skipClearForMVCCSpan && msstw.currSpan <= len(msstw.localKeySpans)) { + // We're either in a local key span, or we're in the first MVCC sstable + // span (before any splits). Add a RangeKeyDel for the whole span. If this + // is the MVCC span, we don't need to keep re-adding it to the fragmenter + // as the fragmenter will take care of splits. Note that currentSpan() + // will return the entire mvcc span in the case we're at an MVCC span. + startKey := storage.EngineKey{Key: msstw.currentSpan().Key}.Encode() + endKey := storage.EngineKey{Key: msstw.currentSpan().EndKey}.Encode() + trailer := pebble.MakeInternalKeyTrailer(0, pebble.InternalKeyKindRangeKeyDelete) + s := rangekey.Span{Start: startKey, End: endKey, Keys: []rangekey.Key{{Trailer: trailer}}} + msstw.rangeKeyFrag.Add(s) } return nil } -func (msstw *multiSSTWriter) finalizeSST(ctx context.Context) error { +func (msstw *multiSSTWriter) finalizeSST(ctx context.Context, nextKey *storage.EngineKey) error { + currSpan := msstw.currentSpan() + if msstw.currSpanIsMVCCSpan() { + // We're in the MVCC span (ie. MVCC / user keys). If skipClearForMVCCSpan + // is true, we don't write a clearRange for the last span at all. Otherwise, + // we need to write a clearRange for all keys leading up to the current key + // we're writing. + currEngineSpan := msstw.mvccSSTSpans[msstw.currSpan-len(msstw.localKeySpans)] + if !msstw.skipClearForMVCCSpan { + if err := msstw.currSST.ClearEngineRange( + currEngineSpan.Start, currEngineSpan.End, + ); err != nil { + msstw.currSST.Close() + return errors.Wrap(err, "failed to clear range on sst file writer") + } + } + } else { + if err := msstw.currSST.ClearRawRange( + currSpan.Key, currSpan.EndKey, + true /* pointKeys */, false, /* rangeKeys */ + ); err != nil { + msstw.currSST.Close() + return errors.Wrap(err, "failed to clear range on sst file writer") + } + } + + // If we're at the last span, call Finish on the fragmenter. If we're not at the + // last span, call Truncate. + if msstw.currSpan == len(msstw.localKeySpans)+len(msstw.mvccSSTSpans)-1 { + msstw.rangeKeyFrag.Finish() + } else { + endKey := storage.EngineKey{Key: currSpan.EndKey} + if msstw.currSpanIsMVCCSpan() { + endKey = msstw.mvccSSTSpans[msstw.currSpan-len(msstw.localKeySpans)].End + } + msstw.rangeKeyFrag.Truncate(endKey.Encode()) + } + err := msstw.currSST.Finish() if err != nil { return errors.Wrap(err, "failed to finish sst") } + if nextKey != nil { + meta := msstw.currSST.Meta + encodedNextKey := nextKey.Encode() + if meta.HasPointKeys && storage.EngineKeyCompare(meta.LargestPoint.UserKey, encodedNextKey) > 0 { + metaEndKey, ok := storage.DecodeEngineKey(meta.LargestPoint.UserKey) + if !ok { + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest point key %s > next sstable start key %s", + meta.LargestPoint.UserKey, nextKey) + } + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest point key %s > next sstable start key %s", + metaEndKey, nextKey) + } + if meta.HasRangeDelKeys && storage.EngineKeyCompare(meta.LargestRangeDel.UserKey, encodedNextKey) > 0 { + metaEndKey, ok := storage.DecodeEngineKey(meta.LargestRangeDel.UserKey) + if !ok { + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest range del %s > next sstable start key %s", + meta.LargestRangeDel.UserKey, nextKey) + } + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest range del %s > next sstable start key %s", + metaEndKey, nextKey) + } + if meta.HasRangeKeys && storage.EngineKeyCompare(meta.LargestRangeKey.UserKey, encodedNextKey) > 0 { + metaEndKey, ok := storage.DecodeEngineKey(meta.LargestRangeKey.UserKey) + if !ok { + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest range key %s > next sstable start key %s", + meta.LargestRangeKey.UserKey, nextKey) + } + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest range key %s > next sstable start key %s", + metaEndKey, nextKey) + } + } msstw.dataSize += msstw.currSST.DataSize msstw.sstSize += int64(msstw.currSST.Meta.Size) msstw.currSpan++ @@ -190,22 +334,22 @@ func (msstw *multiSSTWriter) finalizeSST(ctx context.Context) error { return nil } -// addRangeDelForLastSpan allows us to explicitly add a deletion tombstone -// for the last span in the msstw, if it was instantiated with the expectation +// addClearForMVCCSpan allows us to explicitly add a deletion tombstone +// for the mvcc span in the msstw, if it was instantiated with the expectation // that no tombstone was necessary. -func (msstw *multiSSTWriter) addRangeDelForLastSpan() error { - if !msstw.skipRangeDelForLastSpan { +func (msstw *multiSSTWriter) addClearForMVCCSpan() error { + if !msstw.skipClearForMVCCSpan { // Nothing to do. return nil } - if msstw.currSpan < len(msstw.keySpans)-1 { - // When we switch to the last key span, we will just add a rangedel for it. - // Set skipRangeDelForLastSpan to false. - msstw.skipRangeDelForLastSpan = false + if msstw.currSpan < len(msstw.localKeySpans) { + // When we switch to the mvcc key span, we will just add a rangedel for it. + // Set skipClearForMVCCSpan to false. + msstw.skipClearForMVCCSpan = false return nil } - if msstw.currSpan > len(msstw.keySpans)-1 { - panic("cannot addRangeDel if sst writer has moved past user keys") + if msstw.currSpan >= len(msstw.localKeySpans) { + panic("cannot clearEngineRange if sst writer has moved past user keys") } panic("multiSSTWriter already added keys to sstable that cannot be deleted by a rangedel/rangekeydel within it") } @@ -214,31 +358,57 @@ func (msstw *multiSSTWriter) addRangeDelForLastSpan() error { // writer for writing a point/range key at key. For point keys, endKey and key // must equal each other. func (msstw *multiSSTWriter) rolloverSST( - ctx context.Context, key roachpb.Key, endKey roachpb.Key, + ctx context.Context, key storage.EngineKey, endKey storage.EngineKey, ) error { - for msstw.keySpans[msstw.currSpan].EndKey.Compare(key) <= 0 { + for msstw.currentSpan().EndKey.Compare(key.Key) <= 0 { // Finish the current SST, write to the file, and move to the next key // range. - if err := msstw.finalizeSST(ctx); err != nil { + if err := msstw.finalizeSST(ctx, &key); err != nil { return err } if err := msstw.initSST(ctx); err != nil { return err } } - if msstw.keySpans[msstw.currSpan].Key.Compare(key) > 0 || - msstw.keySpans[msstw.currSpan].EndKey.Compare(endKey) < 0 { - if !key.Equal(endKey) { - return errors.AssertionFailedf("client error: expected %s to fall in one of %s", - roachpb.Span{Key: key, EndKey: endKey}, msstw.keySpans) + currSpan := msstw.currentSpan() + if currSpan.Key.Compare(key.Key) > 0 || currSpan.EndKey.Compare(endKey.Key) < 0 { + if !key.Key.Equal(endKey.Key) { + return errors.AssertionFailedf("client error: expected %s to fall in one of %s or %s", + roachpb.Span{Key: key.Key, EndKey: endKey.Key}, msstw.localKeySpans, msstw.mvccKeySpan) + } + return errors.AssertionFailedf("client error: expected %s to fall in one of %s or %s", key, msstw.localKeySpans, msstw.mvccKeySpan) + } + if msstw.currSpanIsMVCCSpan() && msstw.maxSSTSize > 0 && msstw.currSST.DataSize > msstw.maxSSTSize { + // We're in an MVCC / user keys span, and the current sstable has exceeded + // the max size for MVCC sstables that we should be creating. Split this + // sstable into smaller ones. We do this by splitting the mvccKeySpan + // from [oldStartKey, oldEndKey) to [oldStartKey, key) and [key, oldEndKey). + // The split spans are added to msstw.mvccSSTSpans. + currSpan := &msstw.mvccSSTSpans[msstw.currSpan-len(msstw.localKeySpans)] + if bytes.Equal(currSpan.Start.Key, key.Key) && bytes.Equal(currSpan.Start.Version, key.Version) { + panic("unexpectedly reached max sstable size at start of an mvcc sstable span") + } + oldEndKey := currSpan.End + currSpan.End = key.Copy() + newSpan := storage.EngineKeyRange{Start: currSpan.End, End: oldEndKey} + msstw.mvccSSTSpans = append(msstw.mvccSSTSpans, newSpan) + if msstw.currSpan < len(msstw.localKeySpans)+len(msstw.mvccSSTSpans)-2 { + // This should never happen; we only split sstables when we're at the end + // of mvccSSTSpans. + panic("unexpectedly split an earlier mvcc sstable span in multiSSTWriter") + } + if err := msstw.finalizeSST(ctx, &key); err != nil { + return err + } + if err := msstw.initSST(ctx); err != nil { + return err } - return errors.AssertionFailedf("client error: expected %s to fall in one of %s", key, msstw.keySpans) } return nil } func (msstw *multiSSTWriter) Put(ctx context.Context, key storage.EngineKey, value []byte) error { - if err := msstw.rolloverSST(ctx, key.Key, key.Key); err != nil { + if err := msstw.rolloverSST(ctx, key, key); err != nil { return err } if err := msstw.currSST.PutEngineKey(key, value); err != nil { @@ -254,7 +424,7 @@ func (msstw *multiSSTWriter) PutInternalPointKey( if !ok { return errors.New("cannot decode engine key") } - if err := msstw.rolloverSST(ctx, decodedKey.Key, decodedKey.Key); err != nil { + if err := msstw.rolloverSST(ctx, decodedKey, decodedKey); err != nil { return err } var err error @@ -291,11 +461,14 @@ func decodeRangeStartEnd( } func (msstw *multiSSTWriter) PutInternalRangeDelete(ctx context.Context, start, end []byte) error { + if !msstw.skipClearForMVCCSpan { + panic("can only add internal range deletes to multiSSTWriter if skipClearForMVCCSpan is true") + } decodedStart, decodedEnd, err := decodeRangeStartEnd(start, end) if err != nil { return err } - if err := msstw.rolloverSST(ctx, decodedStart.Key, decodedEnd.Key); err != nil { + if err := msstw.rolloverSST(ctx, decodedStart, decodedEnd); err != nil { return err } if err := msstw.currSST.ClearRawEncodedRange(start, end); err != nil { @@ -307,11 +480,14 @@ func (msstw *multiSSTWriter) PutInternalRangeDelete(ctx context.Context, start, func (msstw *multiSSTWriter) PutInternalRangeKey( ctx context.Context, start, end []byte, key rangekey.Key, ) error { + if !msstw.skipClearForMVCCSpan { + panic("can only add internal range deletes to multiSSTWriter if skipClearForMVCCSpan is true") + } decodedStart, decodedEnd, err := decodeRangeStartEnd(start, end) if err != nil { return err } - if err := msstw.rolloverSST(ctx, decodedStart.Key, decodedEnd.Key); err != nil { + if err := msstw.rolloverSST(ctx, decodedStart, decodedEnd); err != nil { return err } if err := msstw.currSST.PutInternalRangeKey(start, end, key); err != nil { @@ -326,22 +502,33 @@ func (msstw *multiSSTWriter) PutRangeKey( if start.Compare(end) >= 0 { return errors.AssertionFailedf("start key %s must be before end key %s", end, start) } - if err := msstw.rolloverSST(ctx, start, end); err != nil { + if err := msstw.rolloverSST(ctx, storage.EngineKey{Key: start}, storage.EngineKey{Key: end}); err != nil { return err } - if err := msstw.currSST.PutEngineRangeKey(start, end, suffix, value); err != nil { - return errors.Wrap(err, "failed to put range key in sst") + if msstw.skipClearForMVCCSpan { + // Skip the fragmenter. See the comment in skipClearForMVCCSpan. + if err := msstw.currSST.PutEngineRangeKey(start, end, suffix, value); err != nil { + return errors.Wrap(err, "failed to put range key in sst") + } + return nil } + startKey, endKey := storage.EngineKey{Key: start}.Encode(), storage.EngineKey{Key: end}.Encode() + startTrailer := pebble.MakeInternalKeyTrailer(0, pebble.InternalKeyKindRangeKeySet) + msstw.rangeKeyFrag.Add(rangekey.Span{ + Start: startKey, + End: endKey, + Keys: []rangekey.Key{{Trailer: startTrailer, Suffix: suffix, Value: value}}, + }) return nil } func (msstw *multiSSTWriter) Finish(ctx context.Context) (int64, error) { - if msstw.currSpan < len(msstw.keySpans) { + if msstw.currSpan < (len(msstw.localKeySpans) + len(msstw.mvccSSTSpans)) { for { - if err := msstw.finalizeSST(ctx); err != nil { + if err := msstw.finalizeSST(ctx, nil /* nextKey */); err != nil { return 0, err } - if msstw.currSpan >= len(msstw.keySpans) { + if msstw.currSpan >= (len(msstw.localKeySpans) + len(msstw.mvccSSTSpans)) { break } if err := msstw.initSST(ctx); err != nil { @@ -508,8 +695,11 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( // TODO(aaditya): Remove once we support flushableIngests for shared and // external files in the engine. - skipRangeDelForLastSpan := doExcise && (header.SharedReplicate || header.ExternalReplicate) - msstw, err := newMultiSSTWriter(ctx, kvSS.st, kvSS.scratch, keyRanges, kvSS.sstChunkSize, skipRangeDelForLastSpan) + skipClearForMVCCSpan := doExcise && (header.SharedReplicate || header.ExternalReplicate) + // The last key range is the user key span. + localRanges := keyRanges[:len(keyRanges)-1] + mvccRange := keyRanges[len(keyRanges)-1] + msstw, err := newMultiSSTWriter(ctx, kvSS.st, kvSS.scratch, localRanges, mvccRange, kvSS.sstChunkSize, skipClearForMVCCSpan) if err != nil { return noSnap, err } @@ -535,7 +725,7 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( doExcise = false sharedSSTs = nil externalSSTs = nil - if err := msstw.addRangeDelForLastSpan(); err != nil { + if err := msstw.addClearForMVCCSpan(); err != nil { return noSnap, errors.Wrap(err, "adding tombstone for last span") } } @@ -699,7 +889,7 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( sharedSSTs: sharedSSTs, externalSSTs: externalSSTs, doExcise: doExcise, - includesRangeDelForLastSpan: !skipRangeDelForLastSpan, + includesRangeDelForLastSpan: !skipClearForMVCCSpan, clearedSpans: keyRanges, } diff --git a/pkg/storage/engine_key.go b/pkg/storage/engine_key.go index 7cfe689cd2f..dedc828a2e1 100644 --- a/pkg/storage/engine_key.go +++ b/pkg/storage/engine_key.go @@ -363,3 +363,8 @@ type EngineRangeKeyValue struct { Version []byte Value []byte } + +// EngineKeyRange is a key range composed of EngineKeys. +type EngineKeyRange struct { + Start, End EngineKey +} diff --git a/pkg/storage/sst_writer.go b/pkg/storage/sst_writer.go index 9a61dcf4fdd..06b3256798b 100644 --- a/pkg/storage/sst_writer.go +++ b/pkg/storage/sst_writer.go @@ -262,6 +262,17 @@ func (fw *SSTWriter) ClearEngineRangeKey(start, end roachpb.Key, suffix []byte) return fw.fw.RangeKeyUnset(EngineKey{Key: start}.Encode(), EngineKey{Key: end}.Encode(), suffix) } +// ClearEngineRange clears point keys in the specified EngineKey range. +func (fw *SSTWriter) ClearEngineRange(start, end EngineKey) error { + fw.scratch = start.EncodeToBuf(fw.scratch[:0]) + endRaw := end.Encode() + fw.DataSize += int64(len(start.Key)) + int64(len(end.Key)) + if err := fw.fw.DeleteRange(fw.scratch, endRaw); err != nil { + return err + } + return nil +} + // ClearRawEncodedRange implements the InternalWriter interface. func (fw *SSTWriter) ClearRawEncodedRange(start, end []byte) error { startEngine, ok := DecodeEngineKey(start) From 8c760561ec361cf90d2bb363a34f97150b4c5c8f Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Thu, 15 Aug 2024 16:19:10 -0400 Subject: [PATCH 2/2] kvserver: reenable splitting of snapshot sstables This change updates the snapshot strategy's sender side to iterate over points and ranges together, instead of only iterating on points first, then only ranges. This allows us to more efficiently split snapshot sstables on the receiver side. To avoid the need to add a version gate on the receiver side, we propagate a bool, RangeKeysInOrder, to the receiver which is a signal to it to enable sstable splits. Fixes #129026. Epic: none Release note: None --- pkg/cli/debug.go | 9 ++- pkg/kv/kvserver/client_merge_test.go | 2 +- pkg/kv/kvserver/kvserverpb/raft.proto | 6 ++ pkg/kv/kvserver/rditer/replica_data_iter.go | 37 +++++----- .../kvserver/rditer/replica_data_iter_test.go | 21 +++--- .../TestReplicaDataIterator/r1/all/all/output | 6 +- .../r1/all/exclude-user/output | 4 +- .../r1/all/user-only/output | 6 +- .../r1/replicatedOnly/all/output | 4 +- .../r1/replicatedOnly/exclude-user/output | 2 +- .../r1/replicatedOnly/user-only/output | 2 +- .../TestReplicaDataIterator/r2/all/all/output | 6 +- .../r2/all/exclude-user/output | 4 +- .../r2/all/user-only/output | 6 +- .../r2/replicatedOnly/all/output | 4 +- .../r2/replicatedOnly/exclude-user/output | 2 +- .../r2/replicatedOnly/user-only/output | 2 +- .../TestReplicaDataIterator/r3/all/all/output | 6 +- .../r3/all/exclude-user/output | 4 +- .../r3/all/user-only/output | 6 +- .../r3/replicatedOnly/all/output | 4 +- .../r3/replicatedOnly/exclude-user/output | 2 +- .../r3/replicatedOnly/user-only/output | 2 +- pkg/kv/kvserver/replica_command.go | 1 + pkg/kv/kvserver/replica_learner_test.go | 14 ++-- .../replica_sst_snapshot_storage_test.go | 40 ++++++++-- pkg/kv/kvserver/replica_test.go | 3 +- pkg/kv/kvserver/store_snapshot.go | 74 +++++++++++-------- 28 files changed, 157 insertions(+), 122 deletions(-) diff --git a/pkg/cli/debug.go b/pkg/cli/debug.go index 3035b0ed740..2301896fcc2 100644 --- a/pkg/cli/debug.go +++ b/pkg/cli/debug.go @@ -515,10 +515,10 @@ func runDebugRangeData(cmd *cobra.Command, args []string) error { var results int return rditer.IterateReplicaKeySpans(cmd.Context(), &desc, snapshot, debugCtx.replicated, rditer.ReplicatedSpansAll, - func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, _ roachpb.Span) error { for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { - switch keyType { - case storage.IterKeyTypePointsOnly: + hasPoint, hasRange := iter.HasPointAndRange() + if hasPoint { key, err := iter.UnsafeEngineKey() if err != nil { return err @@ -532,8 +532,9 @@ func runDebugRangeData(cmd *cobra.Command, args []string) error { if results == debugCtx.maxResults { return iterutil.StopIteration() } + } - case storage.IterKeyTypeRangesOnly: + if hasRange && iter.RangeKeyChanged() { bounds, err := iter.EngineRangeBounds() if err != nil { return err diff --git a/pkg/kv/kvserver/client_merge_test.go b/pkg/kv/kvserver/client_merge_test.go index 6c03cf822f2..7c0acd26348 100644 --- a/pkg/kv/kvserver/client_merge_test.go +++ b/pkg/kv/kvserver/client_merge_test.go @@ -3903,7 +3903,7 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) { err := rditer.IterateReplicaKeySpans( context.Background(), inSnap.Desc, sendingEngSnapshot, true /* replicatedOnly */, rditer.ReplicatedSpansAll, - func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, span roachpb.Span) error { fw, ok := sstFileWriters[string(span.Key)] if !ok || !fw.span.Equal(span) { return errors.Errorf("unexpected span %s", span) diff --git a/pkg/kv/kvserver/kvserverpb/raft.proto b/pkg/kv/kvserver/kvserverpb/raft.proto index fb2eb61e8cd..3c55df9bfa4 100644 --- a/pkg/kv/kvserver/kvserverpb/raft.proto +++ b/pkg/kv/kvserver/kvserverpb/raft.proto @@ -201,6 +201,12 @@ message SnapshotRequest { // file contents. bool external_replicate = 13; + // If true, the snapshot is iterating over range keys and point + // keys in key order, as opposed to iterating over point keys first + // and range keys second. The receiver can take advantage of this + // to split points/range keys into multiple sstables for ingestion. + bool range_keys_in_order = 14; + reserved 1, 4, 6, 7, 8, 9; } diff --git a/pkg/kv/kvserver/rditer/replica_data_iter.go b/pkg/kv/kvserver/rditer/replica_data_iter.go index c29ba301f4a..93be89dfc3b 100644 --- a/pkg/kv/kvserver/rditer/replica_data_iter.go +++ b/pkg/kv/kvserver/rditer/replica_data_iter.go @@ -403,7 +403,7 @@ func IterateReplicaKeySpans( reader storage.Reader, replicatedOnly bool, replicatedSpansFilter ReplicatedSpansFilter, - visitor func(storage.EngineIterator, roachpb.Span, storage.IterKeyType) error, + visitor func(storage.EngineIterator, roachpb.Span) error, ) error { if !reader.ConsistentIterators() { panic("reader must provide consistent iterators") @@ -425,28 +425,25 @@ func IterateReplicaKeySpans( UnreplicatedByRangeID: true, }) } - keyTypes := []storage.IterKeyType{storage.IterKeyTypePointsOnly, storage.IterKeyTypeRangesOnly} for _, span := range spans { - for _, keyType := range keyTypes { - err := func() error { - iter, err := reader.NewEngineIterator(ctx, storage.IterOptions{ - KeyTypes: keyType, - LowerBound: span.Key, - UpperBound: span.EndKey, - }) - if err != nil { - return err - } - defer iter.Close() - ok, err := iter.SeekEngineKeyGE(storage.EngineKey{Key: span.Key}) - if err == nil && ok { - err = visitor(iter, span, keyType) - } - return err - }() + err := func() error { + iter, err := reader.NewEngineIterator(ctx, storage.IterOptions{ + KeyTypes: storage.IterKeyTypePointsAndRanges, + LowerBound: span.Key, + UpperBound: span.EndKey, + }) if err != nil { - return iterutil.Map(err) + return err } + defer iter.Close() + ok, err := iter.SeekEngineKeyGE(storage.EngineKey{Key: span.Key}) + if err == nil && ok { + err = visitor(iter, span) + } + return err + }() + if err != nil { + return iterutil.Map(err) } } return nil diff --git a/pkg/kv/kvserver/rditer/replica_data_iter_test.go b/pkg/kv/kvserver/rditer/replica_data_iter_test.go index 859478669de..a1de1020650 100644 --- a/pkg/kv/kvserver/rditer/replica_data_iter_test.go +++ b/pkg/kv/kvserver/rditer/replica_data_iter_test.go @@ -175,7 +175,7 @@ func verifyIterateReplicaKeySpans( require.NoError(t, IterateReplicaKeySpans(context.Background(), desc, readWriter, replicatedOnly, replicatedSpansFilter, - func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, span roachpb.Span) error { var err error for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { // Span should not be empty. @@ -186,8 +186,8 @@ func verifyIterateReplicaKeySpans( require.True(t, span.ContainsKey(key.Key), "%s not in %s", key, span) require.True(t, key.IsLockTableKey() || key.IsMVCCKey(), "%s neither lock nor MVCC", key) - switch keyType { - case storage.IterKeyTypePointsOnly: + hasPoint, hasRange := iter.HasPointAndRange() + if hasPoint { var mvccKey storage.MVCCKey if key.IsMVCCKey() { var err error @@ -213,8 +213,8 @@ func verifyIterateReplicaKeySpans( fmt.Sprintf("%x", key.Version), mvccKey.String(), }) - - case storage.IterKeyTypeRangesOnly: + } + if hasRange && iter.RangeKeyChanged() { bounds, err := iter.EngineRangeBounds() require.NoError(t, err) require.True(t, span.Contains(bounds), "%s not contained in %s", bounds, span) @@ -234,9 +234,6 @@ func verifyIterateReplicaKeySpans( mvccRangeKey.String(), }) } - - default: - t.Fatalf("unexpected key type %v", keyType) } } return err @@ -482,9 +479,11 @@ func TestReplicaDataIteratorGlobalRangeKey(t *testing.T) { var actualSpans []roachpb.Span require.NoError(t, IterateReplicaKeySpans( context.Background(), &desc, snapshot, replicatedOnly, ReplicatedSpansAll, - func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, span roachpb.Span) error { // We should never see any point keys. - require.Equal(t, storage.IterKeyTypeRangesOnly, keyType) + hasPoint, hasRange := iter.HasPointAndRange() + require.False(t, hasPoint) + require.True(t, hasRange) // The iterator should already be positioned on the range key, which should // span the entire key span and be the only range key. @@ -590,7 +589,7 @@ func benchReplicaEngineDataIterator(b *testing.B, numRanges, numKeysPerRange, va for _, desc := range descs { err := IterateReplicaKeySpans( context.Background(), &desc, snapshot, false /* replicatedOnly */, ReplicatedSpansAll, - func(iter storage.EngineIterator, _ roachpb.Span, _ storage.IterKeyType) error { + func(iter storage.EngineIterator, _ roachpb.Span) error { var err error for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { _, _ = iter.UnsafeEngineKey() diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output index a70c35c621b..650a35b2e8b 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output @@ -3,25 +3,25 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{u""-v""} | 0169897572667462 | | | /Local/RangeID/1/u/RangeTombstone | | /Local/RangeID/1/{u""-v""} | 0169897572667468 | | | /Local/RangeID/1/u/RaftHardState | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000001 | | | /Local/RangeID/1/u/RaftLog/logIndex:1 | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000002 | | | /Local/RangeID/1/u/RaftLog/logIndex:2 | | /Local/RangeID/1/{u""-v""} | 01698975726c7274 | | | /Local/RangeID/1/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000172647363 | | 0000000000000001 | /Local/Range"a"/RangeDescriptor/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{a"-b"} | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{a"-b"} | 016b1261ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"a\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{a"-b"} | 017a6b12016b126100ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"a"/RangeDescriptor | | /Local/Lock"{a"-b"} | 017a6b12610001 | | 030ce61c175eb445878c36dcf4062ada4c | "a" | +| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | | {a-b} | 61 | | 0000000000000001 | "a"/0.000000001,0 | | {a-b} | 61ffffffff | | 0000000000000001 | "a\xff\xff\xff\xff"/0.000000001,0 | -| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/exclude-user/output index d4751da1f82..89612c49d94 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/exclude-user/output @@ -3,18 +3,18 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{u""-v""} | 0169897572667462 | | | /Local/RangeID/1/u/RangeTombstone | | /Local/RangeID/1/{u""-v""} | 0169897572667468 | | | /Local/RangeID/1/u/RaftHardState | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000001 | | | /Local/RangeID/1/u/RaftLog/logIndex:1 | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000002 | | | /Local/RangeID/1/u/RaftLog/logIndex:2 | | /Local/RangeID/1/{u""-v""} | 01698975726c7274 | | | /Local/RangeID/1/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000172647363 | | 0000000000000001 | /Local/Range"a"/RangeDescriptor/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{a"-b"} | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/user-only/output index 21d8f060459..bef327ba62d 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/user-only/output @@ -3,19 +3,19 @@ echo +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{u""-v""} | 0169897572667462 | | | /Local/RangeID/1/u/RangeTombstone | | /Local/RangeID/1/{u""-v""} | 0169897572667468 | | | /Local/RangeID/1/u/RaftHardState | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000001 | | | /Local/RangeID/1/u/RaftLog/logIndex:1 | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000002 | | | /Local/RangeID/1/u/RaftLog/logIndex:2 | | /Local/RangeID/1/{u""-v""} | 01698975726c7274 | | | /Local/RangeID/1/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | +| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | | {a-b} | 61 | | 0000000000000001 | "a"/0.000000001,0 | | {a-b} | 61ffffffff | | 0000000000000001 | "a\xff\xff\xff\xff"/0.000000001,0 | -| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/all/output index b487074ffe6..107ed05a398 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/all/output @@ -3,19 +3,19 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000172647363 | | 0000000000000001 | /Local/Range"a"/RangeDescriptor/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{a"-b"} | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{a"-b"} | 016b1261ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"a\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{a"-b"} | 017a6b12016b126100ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"a"/RangeDescriptor | | /Local/Lock"{a"-b"} | 017a6b12610001 | | 030ce61c175eb445878c36dcf4062ada4c | "a" | +| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | | {a-b} | 61 | | 0000000000000001 | "a"/0.000000001,0 | | {a-b} | 61ffffffff | | 0000000000000001 | "a\xff\xff\xff\xff"/0.000000001,0 | -| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/exclude-user/output index fc247f4b9f6..0c6052519a7 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/exclude-user/output @@ -3,12 +3,12 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000172647363 | | 0000000000000001 | /Local/Range"a"/RangeDescriptor/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{a"-b"} | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/user-only/output index ffd95d05263..a6b80632c58 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/user-only/output @@ -3,7 +3,7 @@ echo +-------+------------+------------+--------------------+-----------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +-------+------------+------------+--------------------+-----------------------------------+ +| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | | {a-b} | 61 | | 0000000000000001 | "a"/0.000000001,0 | | {a-b} | 61ffffffff | | 0000000000000001 | "a\xff\xff\xff\xff"/0.000000001,0 | -| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | +-------+------------+------------+--------------------+-----------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/all/output index 1c3cb3da75b..c4adc4e4727 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/all/output @@ -3,25 +3,25 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{u""-v""} | 01698a7572667462 | | | /Local/RangeID/2/u/RangeTombstone | | /Local/RangeID/2/{u""-v""} | 01698a7572667468 | | | /Local/RangeID/2/u/RaftHardState | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000001 | | | /Local/RangeID/2/u/RaftLog/logIndex:1 | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000002 | | | /Local/RangeID/2/u/RaftLog/logIndex:2 | | /Local/RangeID/2/{u""-v""} | 01698a75726c7274 | | | /Local/RangeID/2/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000172647363 | | 0000000000000001 | /Local/Range"b"/RangeDescriptor/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"b"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{b"-c"} | 016b126200ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"b\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{b"-c"} | 016b1262ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"b\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{b"-c"} | 017a6b12016b126200ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"b"/RangeDescriptor | | /Local/Lock"{b"-c"} | 017a6b12620001 | | 030ce61c175eb445878c36dcf4062ada4c | "b" | +| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | | {b-c} | 62 | | 0000000000000001 | "b"/0.000000001,0 | | {b-c} | 62ffffffff | | 0000000000000001 | "b\xff\xff\xff\xff"/0.000000001,0 | -| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/exclude-user/output index 936fb93c42d..29775b06a64 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/exclude-user/output @@ -3,18 +3,18 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{u""-v""} | 01698a7572667462 | | | /Local/RangeID/2/u/RangeTombstone | | /Local/RangeID/2/{u""-v""} | 01698a7572667468 | | | /Local/RangeID/2/u/RaftHardState | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000001 | | | /Local/RangeID/2/u/RaftLog/logIndex:1 | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000002 | | | /Local/RangeID/2/u/RaftLog/logIndex:2 | | /Local/RangeID/2/{u""-v""} | 01698a75726c7274 | | | /Local/RangeID/2/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000172647363 | | 0000000000000001 | /Local/Range"b"/RangeDescriptor/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"b"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{b"-c"} | 016b126200ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"b\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/user-only/output index c8f0a593037..c518246b0aa 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/user-only/output @@ -3,19 +3,19 @@ echo +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{u""-v""} | 01698a7572667462 | | | /Local/RangeID/2/u/RangeTombstone | | /Local/RangeID/2/{u""-v""} | 01698a7572667468 | | | /Local/RangeID/2/u/RaftHardState | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000001 | | | /Local/RangeID/2/u/RaftLog/logIndex:1 | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000002 | | | /Local/RangeID/2/u/RaftLog/logIndex:2 | | /Local/RangeID/2/{u""-v""} | 01698a75726c7274 | | | /Local/RangeID/2/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | +| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | | {b-c} | 62 | | 0000000000000001 | "b"/0.000000001,0 | | {b-c} | 62ffffffff | | 0000000000000001 | "b\xff\xff\xff\xff"/0.000000001,0 | -| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/all/output index c338a02b844..256b75a810d 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/all/output @@ -3,19 +3,19 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000172647363 | | 0000000000000001 | /Local/Range"b"/RangeDescriptor/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"b"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{b"-c"} | 016b126200ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"b\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{b"-c"} | 016b1262ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"b\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{b"-c"} | 017a6b12016b126200ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"b"/RangeDescriptor | | /Local/Lock"{b"-c"} | 017a6b12620001 | | 030ce61c175eb445878c36dcf4062ada4c | "b" | +| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | | {b-c} | 62 | | 0000000000000001 | "b"/0.000000001,0 | | {b-c} | 62ffffffff | | 0000000000000001 | "b\xff\xff\xff\xff"/0.000000001,0 | -| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/exclude-user/output index 83056687a5d..3d0d6f5a9f4 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/exclude-user/output @@ -3,12 +3,12 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000172647363 | | 0000000000000001 | /Local/Range"b"/RangeDescriptor/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"b"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{b"-c"} | 016b126200ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"b\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/user-only/output index c70d703437d..d66434a8982 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/user-only/output @@ -3,7 +3,7 @@ echo +-------+------------+------------+--------------------+-----------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +-------+------------+------------+--------------------+-----------------------------------+ +| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | | {b-c} | 62 | | 0000000000000001 | "b"/0.000000001,0 | | {b-c} | 62ffffffff | | 0000000000000001 | "b\xff\xff\xff\xff"/0.000000001,0 | -| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | +-------+------------+------------+--------------------+-----------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/all/output index a143fa468e1..7226bcc6523 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/all/output @@ -3,25 +3,25 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{u""-v""} | 01698b7572667462 | | | /Local/RangeID/3/u/RangeTombstone | | /Local/RangeID/3/{u""-v""} | 01698b7572667468 | | | /Local/RangeID/3/u/RaftHardState | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000001 | | | /Local/RangeID/3/u/RaftLog/logIndex:1 | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000002 | | | /Local/RangeID/3/u/RaftLog/logIndex:2 | | /Local/RangeID/3/{u""-v""} | 01698b75726c7274 | | | /Local/RangeID/3/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000172647363 | | 0000000000000001 | /Local/Range"c"/RangeDescriptor/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"c"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{c"-d"} | 016b126300ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"c\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{c"-d"} | 016b1263ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"c\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{c"-d"} | 017a6b12016b126300ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"c"/RangeDescriptor | | /Local/Lock"{c"-d"} | 017a6b12630001 | | 030ce61c175eb445878c36dcf4062ada4c | "c" | +| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | | {c-d} | 63 | | 0000000000000001 | "c"/0.000000001,0 | | {c-d} | 63ffffffff | | 0000000000000001 | "c\xff\xff\xff\xff"/0.000000001,0 | -| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/exclude-user/output index 396aafe31e6..4994b21a3cd 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/exclude-user/output @@ -3,18 +3,18 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{u""-v""} | 01698b7572667462 | | | /Local/RangeID/3/u/RangeTombstone | | /Local/RangeID/3/{u""-v""} | 01698b7572667468 | | | /Local/RangeID/3/u/RaftHardState | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000001 | | | /Local/RangeID/3/u/RaftLog/logIndex:1 | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000002 | | | /Local/RangeID/3/u/RaftLog/logIndex:2 | | /Local/RangeID/3/{u""-v""} | 01698b75726c7274 | | | /Local/RangeID/3/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000172647363 | | 0000000000000001 | /Local/Range"c"/RangeDescriptor/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"c"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{c"-d"} | 016b126300ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"c\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/user-only/output index e146d6ec8b8..aad38dbbcb5 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/user-only/output @@ -3,19 +3,19 @@ echo +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{u""-v""} | 01698b7572667462 | | | /Local/RangeID/3/u/RangeTombstone | | /Local/RangeID/3/{u""-v""} | 01698b7572667468 | | | /Local/RangeID/3/u/RaftHardState | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000001 | | | /Local/RangeID/3/u/RaftLog/logIndex:1 | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000002 | | | /Local/RangeID/3/u/RaftLog/logIndex:2 | | /Local/RangeID/3/{u""-v""} | 01698b75726c7274 | | | /Local/RangeID/3/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | +| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | | {c-d} | 63 | | 0000000000000001 | "c"/0.000000001,0 | | {c-d} | 63ffffffff | | 0000000000000001 | "c\xff\xff\xff\xff"/0.000000001,0 | -| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/all/output index a0a0b86d22b..53a85f4757a 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/all/output @@ -3,19 +3,19 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000172647363 | | 0000000000000001 | /Local/Range"c"/RangeDescriptor/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"c"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{c"-d"} | 016b126300ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"c\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{c"-d"} | 016b1263ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"c\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{c"-d"} | 017a6b12016b126300ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"c"/RangeDescriptor | | /Local/Lock"{c"-d"} | 017a6b12630001 | | 030ce61c175eb445878c36dcf4062ada4c | "c" | +| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | | {c-d} | 63 | | 0000000000000001 | "c"/0.000000001,0 | | {c-d} | 63ffffffff | | 0000000000000001 | "c\xff\xff\xff\xff"/0.000000001,0 | -| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/exclude-user/output index ac831e72ded..22b8ec31e25 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/exclude-user/output @@ -3,12 +3,12 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000172647363 | | 0000000000000001 | /Local/Range"c"/RangeDescriptor/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"c"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{c"-d"} | 016b126300ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"c\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/user-only/output index 9949d61859e..55c42ef5723 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/user-only/output @@ -3,7 +3,7 @@ echo +-------+------------+------------+--------------------+-----------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +-------+------------+------------+--------------------+-----------------------------------+ +| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | | {c-d} | 63 | | 0000000000000001 | "c"/0.000000001,0 | | {c-d} | 63ffffffff | | 0000000000000001 | "c\xff\xff\xff\xff"/0.000000001,0 | -| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | +-------+------------+------------+--------------------+-----------------------------------+ diff --git a/pkg/kv/kvserver/replica_command.go b/pkg/kv/kvserver/replica_command.go index 5963bbc7391..babb6ae8ab2 100644 --- a/pkg/kv/kvserver/replica_command.go +++ b/pkg/kv/kvserver/replica_command.go @@ -3317,6 +3317,7 @@ func (r *Replica) followerSendSnapshot( SenderQueuePriority: req.SenderQueuePriority, SharedReplicate: sharedReplicate, ExternalReplicate: externalReplicate, + RangeKeysInOrder: true, } newBatchFn := func() storage.WriteBatch { return r.store.TODOEngine().NewWriteBatch() diff --git a/pkg/kv/kvserver/replica_learner_test.go b/pkg/kv/kvserver/replica_learner_test.go index 64c592d9d9a..7854358a34a 100644 --- a/pkg/kv/kvserver/replica_learner_test.go +++ b/pkg/kv/kvserver/replica_learner_test.go @@ -2232,11 +2232,12 @@ func getExpectedSnapshotSizeBytes( err = rditer.IterateReplicaKeySpans( ctx, snap.State.Desc, snap.EngineSnap, true /* replicatedOnly */, rditer.ReplicatedSpansAll, - func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, _ roachpb.Span) error { var err error for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { - switch keyType { - case storage.IterKeyTypePointsOnly: + hasPoint, hasRange := iter.HasPointAndRange() + + if hasPoint { unsafeKey, err := iter.UnsafeEngineKey() if err != nil { return err @@ -2248,8 +2249,8 @@ func getExpectedSnapshotSizeBytes( if err := b.PutEngineKey(unsafeKey, v); err != nil { return err } - - case storage.IterKeyTypeRangesOnly: + } + if hasRange && iter.RangeKeyChanged() { bounds, err := iter.EngineRangeBounds() if err != nil { return err @@ -2260,9 +2261,6 @@ func getExpectedSnapshotSizeBytes( return err } } - - default: - return errors.Errorf("unexpected key type %v", keyType) } } return err diff --git a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go index 6e13695a7e9..3757096942a 100644 --- a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go +++ b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go @@ -12,6 +12,7 @@ package kvserver import ( "context" + "encoding/binary" "fmt" io "io" "path/filepath" @@ -28,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" "github.com/cockroachdb/errors/oserror" @@ -283,7 +285,7 @@ func TestMultiSSTWriterInitSST(t *testing.T) { msstw, err := newMultiSSTWriter( ctx, cluster.MakeTestingClusterSettings(), scratch, localSpans, mvccSpan, 0, - false, /* skipRangeDelForMVCCSpan */ + false /* skipRangeDelForMVCCSpan */, false, /* rangeKeysInOrder */ ) require.NoError(t, err) _, err = msstw.Finish(ctx) @@ -365,16 +367,28 @@ func TestMultiSSTWriterSize(t *testing.T) { mvccSpan := keySpans[len(keySpans)-1] // Make a reference msstw with the default size. - referenceMsstw, err := newMultiSSTWriter(ctx, settings, ref, localSpans, mvccSpan, 0, false) + referenceMsstw, err := newMultiSSTWriter(ctx, settings, ref, localSpans, mvccSpan, 0, false, true /* rangeKeysInOrder */) require.NoError(t, err) require.Equal(t, int64(0), referenceMsstw.dataSize) + now := timeutil.Now().UnixNano() for i := range localSpans { require.NoError(t, referenceMsstw.Put(ctx, storage.EngineKey{Key: localSpans[i].Key}, []byte("foo"))) } - for i := 0; i < 100; i++ { - require.NoError(t, referenceMsstw.Put(ctx, storage.EngineKey{Key: roachpb.Key(append(desc.StartKey, byte(i)))}, []byte("foobarbaz"))) + for i := 0; i < 1000; i++ { + key := binary.BigEndian.AppendUint32(append([]byte(nil), desc.StartKey...), uint32(i)) + mvccKey := storage.MVCCKey{Key: roachpb.Key(key), Timestamp: hlc.Timestamp{WallTime: now}} + engineKey, ok := storage.DecodeEngineKey(storage.EncodeMVCCKey(mvccKey)) + require.True(t, ok) + + if i%50 == 0 { + // Add a range key. + endKey := binary.BigEndian.AppendUint32(desc.StartKey, uint32(i+10)) + require.NoError(t, referenceMsstw.PutRangeKey( + ctx, key, endKey, storage.EncodeMVCCTimestampSuffix(mvccKey.Timestamp.WallPrev()), []byte(""))) + } + require.NoError(t, referenceMsstw.Put(ctx, engineKey, []byte("foobarbaz"))) } _, err = referenceMsstw.Finish(ctx) require.NoError(t, err) @@ -385,7 +399,7 @@ func TestMultiSSTWriterSize(t *testing.T) { MaxSnapshotSSTableSize.Override(ctx, &settings.SV, 100) - multiSSTWriter, err := newMultiSSTWriter(ctx, settings, scratch, localSpans, mvccSpan, 0, false) + multiSSTWriter, err := newMultiSSTWriter(ctx, settings, scratch, localSpans, mvccSpan, 0, false, true /* rangeKeysInOrder */) require.NoError(t, err) require.Equal(t, int64(0), multiSSTWriter.dataSize) @@ -393,8 +407,18 @@ func TestMultiSSTWriterSize(t *testing.T) { require.NoError(t, multiSSTWriter.Put(ctx, storage.EngineKey{Key: localSpans[i].Key}, []byte("foo"))) } - for i := 0; i < 100; i++ { - require.NoError(t, multiSSTWriter.Put(ctx, storage.EngineKey{Key: roachpb.Key(append(desc.StartKey, byte(i)))}, []byte("foobarbaz"))) + for i := 0; i < 1000; i++ { + key := binary.BigEndian.AppendUint32(append([]byte(nil), desc.StartKey...), uint32(i)) + mvccKey := storage.MVCCKey{Key: roachpb.Key(key), Timestamp: hlc.Timestamp{WallTime: now}} + engineKey, ok := storage.DecodeEngineKey(storage.EncodeMVCCKey(mvccKey)) + require.True(t, ok) + if i%50 == 0 { + // Add a range key. + endKey := binary.BigEndian.AppendUint32(desc.StartKey, uint32(i+10)) + require.NoError(t, multiSSTWriter.PutRangeKey( + ctx, key, endKey, storage.EncodeMVCCTimestampSuffix(mvccKey.Timestamp.WallPrev()), []byte(""))) + } + require.NoError(t, multiSSTWriter.Put(ctx, engineKey, []byte("foobarbaz"))) } _, err = multiSSTWriter.Finish(ctx) @@ -462,7 +486,7 @@ func TestMultiSSTWriterAddLastSpan(t *testing.T) { msstw, err := newMultiSSTWriter( ctx, cluster.MakeTestingClusterSettings(), scratch, localSpans, mvccSpan, 0, - true, /* skipRangeDelForMVCCSpan */ + true /* skipRangeDelForMVCCSpan */, false, /* rangeKeysInOrder */ ) require.NoError(t, err) if addRangeDel { diff --git a/pkg/kv/kvserver/replica_test.go b/pkg/kv/kvserver/replica_test.go index deb88e08950..be603fcd80d 100644 --- a/pkg/kv/kvserver/replica_test.go +++ b/pkg/kv/kvserver/replica_test.go @@ -7203,8 +7203,7 @@ func TestReplicaDestroy(t *testing.T) { require.NoError(t, rditer.IterateReplicaKeySpans( ctx, tc.repl.Desc(), engSnapshot, false /* replicatedOnly */, rditer.ReplicatedSpansAll, - func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error { - require.Equal(t, storage.IterKeyTypePointsOnly, keyType) + func(iter storage.EngineIterator, _ roachpb.Span) error { var err error for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { key, err := iter.UnsafeEngineKey() diff --git a/pkg/kv/kvserver/store_snapshot.go b/pkg/kv/kvserver/store_snapshot.go index 3e5cfed4091..2e4b939b578 100644 --- a/pkg/kv/kvserver/store_snapshot.go +++ b/pkg/kv/kvserver/store_snapshot.go @@ -36,6 +36,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/metamorphic" "github.com/cockroachdb/cockroach/pkg/util/metric" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -71,8 +72,13 @@ var MaxSnapshotSSTableSize = settings.RegisterByteSizeSetting( settings.SystemOnly, "kv.snapshot_rebalance.max_sst_size", "maximum size of a rebalance or recovery SST size", - 128<<20, // 128 MB - settings.PositiveInt, + int64(metamorphic.ConstantWithTestRange( + "kv.snapshot_rebalance.max_sst_size", + 128<<20, /* defaultValue */ + 32<<10, /* metamorphic min */ + 512<<20, /* metamorphic max */ + )), // 128 MB default + settings.NonNegativeInt, ) // snapshotMetrics contains metrics on the number and size of snapshots in @@ -176,6 +182,7 @@ func newMultiSSTWriter( mvccKeySpan roachpb.Span, sstChunkSize int64, skipClearForMVCCSpan bool, + rangeKeysInOrder bool, ) (*multiSSTWriter, error) { msstw := &multiSSTWriter{ st: st, @@ -189,10 +196,17 @@ func newMultiSSTWriter( sstChunkSize: sstChunkSize, skipClearForMVCCSpan: skipClearForMVCCSpan, } - if !skipClearForMVCCSpan { + if !skipClearForMVCCSpan && rangeKeysInOrder { // If skipClearForMVCCSpan is true, we don't split the MVCC span across // multiple sstables, as addClearForMVCCSpan could be called by the caller // at any time. + // + // We also disable snapshot sstable splitting unless the sender has + // specified in its snapshot header that it is sending range keys in + // key order alongside point keys, as opposed to sending them at the end + // of the snapshot. This is necessary to efficiently produce fragmented + // snapshot sstables, as otherwise range keys will arrive out-of-order + // wrt. point keys. msstw.maxSSTSize = MaxSnapshotSSTableSize.Get(&st.SV) } msstw.rangeKeyFrag = rangekey.Fragmenter{ @@ -699,7 +713,7 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( // The last key range is the user key span. localRanges := keyRanges[:len(keyRanges)-1] mvccRange := keyRanges[len(keyRanges)-1] - msstw, err := newMultiSSTWriter(ctx, kvSS.st, kvSS.scratch, localRanges, mvccRange, kvSS.sstChunkSize, skipClearForMVCCSpan) + msstw, err := newMultiSSTWriter(ctx, kvSS.st, kvSS.scratch, localRanges, mvccRange, kvSS.sstChunkSize, skipClearForMVCCSpan, header.RangeKeysInOrder) if err != nil { return noSnap, err } @@ -980,14 +994,33 @@ func (kvSS *kvBatchSnapshotStrategy) Send( replicatedFilter = rditer.ReplicatedSpansExcludeUser } - iterateRKSpansVisitor := func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error { + iterateRKSpansVisitor := func(iter storage.EngineIterator, _ roachpb.Span) error { timingTag.start("iter") defer timingTag.stop("iter") var err error - switch keyType { - case storage.IterKeyTypePointsOnly: - for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { + for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { + hasPoint, hasRange := iter.HasPointAndRange() + if hasRange && iter.RangeKeyChanged() { + bounds, err := iter.EngineRangeBounds() + if err != nil { + return err + } + for _, rkv := range iter.EngineRangeKeys() { + rangeKVs++ + if b == nil { + b = kvSS.newWriteBatch() + } + err := b.PutEngineRangeKey(bounds.Key, bounds.EndKey, rkv.Version, rkv.Value) + if err != nil { + return err + } + if err = maybeFlushBatch(); err != nil { + return err + } + } + } + if hasPoint { kvs++ if b == nil { b = kvSS.newWriteBatch() @@ -1007,30 +1040,6 @@ func (kvSS *kvBatchSnapshotStrategy) Send( return err } } - - case storage.IterKeyTypeRangesOnly: - for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { - bounds, err := iter.EngineRangeBounds() - if err != nil { - return err - } - for _, rkv := range iter.EngineRangeKeys() { - rangeKVs++ - if b == nil { - b = kvSS.newWriteBatch() - } - err := b.PutEngineRangeKey(bounds.Key, bounds.EndKey, rkv.Version, rkv.Value) - if err != nil { - return err - } - if err = maybeFlushBatch(); err != nil { - return err - } - } - } - - default: - return errors.AssertionFailedf("unexpected key type %v", keyType) } return err } @@ -1953,6 +1962,7 @@ func SendEmptySnapshot( State: state, RaftMessageRequest: req, RangeSize: ms.Total(), + RangeKeysInOrder: true, } stream, err := NewMultiRaftClient(cc).RaftSnapshot(ctx)