release-23.2: bump Pebble and split snapshot sstables

DEPS.bzl

@@ -1569,10 +1569,10 @@ def go_deps():
         patches = [
             "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
         ],
-        sha256 = "2375bf475d4f7800aae8fe4ca45c0c94f8df104df55c0eb2024ed99631fff350",
-        strip_prefix = "github.com/cockroachdb/[email protected].20240729165622-7e594e283744",
+        sha256 = "5e70be03a716241f86210ab25bc511d0ec3d0ee15984a897d9ae6535266876d3",
+        strip_prefix = "github.com/cockroachdb/[email protected].20241107030133-19b47dc92d30",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v1.1.2-0.20240729165622-7e594e283744.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v1.1.2-0.20241107030133-19b47dc92d30.zip",
         ],
     )
     go_repository(

build/bazelutil/distdir_files.bzl

@@ -318,7 +318,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/go-test-teamcity/com_github_cockroachdb_go_test_teamcity-v0.0.0-20191211140407-cff980ad0a55.zip": "bac30148e525b79d004da84d16453ddd2d5cd20528e9187f1d7dac708335674b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20230118201751-21c54148d20b.zip": "ca7776f47e5fecb4c495490a679036bfc29d95bd7625290cfdb9abb0baf97476",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v1.1.2-0.20240729165622-7e594e283744.zip": "2375bf475d4f7800aae8fe4ca45c0c94f8df104df55c0eb2024ed99631fff350",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v1.1.2-0.20241107030133-19b47dc92d30.zip": "5e70be03a716241f86210ab25bc511d0ec3d0ee15984a897d9ae6535266876d3",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/raft/v3/com_github_cockroachdb_raft_v3-v3.0.0-20240612143628-49a8a628fa47.zip": "ce7085b92c505bec2c778fcac5d186a3156b1f0e6483316b53713f0f329f82d4",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.5.zip": "11b30528eb0dafc8bc1a5ba39d81277c257cbe6946a7564402f588357c164560",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",

go.mod

@@ -113,7 +113,7 @@ require (
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
 	github.com/cockroachdb/gostdlib v1.19.0
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b
-	github.com/cockroachdb/pebble v1.1.2-0.20240729165622-7e594e283744
+	github.com/cockroachdb/pebble v1.1.2-0.20241107030133-19b47dc92d30
 	github.com/cockroachdb/redact v1.1.5
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

go.sum

@@ -482,8 +482,8 @@ github.com/cockroachdb/gostdlib v1.19.0/go.mod h1:+dqqpARXbE/gRDEhCak6dm0l14AaTy
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v1.1.2-0.20240729165622-7e594e283744 h1:9urOSbgK9+hKysVr5eqmAwgxCbrOs6iS1p+aK2b17Ws=
-github.com/cockroachdb/pebble v1.1.2-0.20240729165622-7e594e283744/go.mod h1:CnHUPUo2XSUfriPdNcyweFtE9dnZMSU8Gt10sXRGoJg=
+github.com/cockroachdb/pebble v1.1.2-0.20241107030133-19b47dc92d30 h1:QG63MGk228WUsCIbeT/bEWXqwkx8I+RNKH7fzGTZ6M8=
+github.com/cockroachdb/pebble v1.1.2-0.20241107030133-19b47dc92d30/go.mod h1:CnHUPUo2XSUfriPdNcyweFtE9dnZMSU8Gt10sXRGoJg=
 github.com/cockroachdb/raft/v3 v3.0.0-20240612143628-49a8a628fa47 h1:fT0adzNbw60Pd/iMkgz+Eq5gEJTqyZ9SnNlvIYjg22A=
 github.com/cockroachdb/raft/v3 v3.0.0-20240612143628-49a8a628fa47/go.mod h1:tP6U+sRzrl75ltgmFcdZg9reZVEyM3vKTxAWmwpHtB8=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=

pkg/cli/debug.go

@@ -474,10 +474,10 @@ func runDebugRangeData(cmd *cobra.Command, args []string) error {
 
 	var results int
 	return rditer.IterateReplicaKeySpans(&desc, snapshot, debugCtx.replicated, rditer.ReplicatedSpansAll,
-		func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error {
+		func(iter storage.EngineIterator, _ roachpb.Span) error {
 			for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() {
-				switch keyType {
-				case storage.IterKeyTypePointsOnly:
+				hasPoint, hasRange := iter.HasPointAndRange()
+				if hasPoint {
 					key, err := iter.UnsafeEngineKey()
 					if err != nil {
 						return err
@@ -491,8 +491,9 @@ func runDebugRangeData(cmd *cobra.Command, args []string) error {
 					if results == debugCtx.maxResults {
 						return iterutil.StopIteration()
 					}
+				}
 
-				case storage.IterKeyTypeRangesOnly:
+				if hasRange && iter.RangeKeyChanged() {
 					bounds, err := iter.EngineRangeBounds()
 					if err != nil {
 						return err

pkg/kv/kvserver/BUILD.bazel

@@ -512,6 +512,8 @@ go_test(
         "@com_github_cockroachdb_errors//oserror",
         "@com_github_cockroachdb_logtags//:logtags",
         "@com_github_cockroachdb_pebble//:pebble",
+        "@com_github_cockroachdb_pebble//sstable",
+        "@com_github_cockroachdb_pebble//vfs",
         "@com_github_cockroachdb_redact//:redact",
         "@com_github_dustin_go_humanize//:go-humanize",
         "@com_github_gogo_protobuf//proto",

pkg/kv/kvserver/client_merge_test.go

@@ -3813,7 +3813,7 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) {
 		}
 
 		err := rditer.IterateReplicaKeySpans(inSnap.Desc, sendingEngSnapshot, true /* replicatedOnly */, rditer.ReplicatedSpansAll,
-			func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error {
+			func(iter storage.EngineIterator, span roachpb.Span) error {
 				fw, ok := sstFileWriters[string(span.Key)]
 				if !ok || !fw.span.Equal(span) {
 					return errors.Errorf("unexpected span %s", span)

pkg/kv/kvserver/kvserverpb/raft.proto

@@ -250,6 +250,17 @@ message SnapshotRequest {
     // metadata present in the snapshot, but not file contents.
     bool shared_replicate = 12;
 
+    // If true, the snapshot could contain external files. Such files
+    // will have their metadata present in the snapshot but not the
+    // file contents.
+    bool external_replicate = 13;
+
+    // If true, the snapshot is iterating over range keys and point
+    // keys in key order, as opposed to iterating over point keys first
+    // and range keys second. The receiver can take advantage of this
+    // to split points/range keys into multiple sstables for ingestion.
+    bool range_keys_in_order = 14;
+
     reserved 1, 4;
   }
 

pkg/kv/kvserver/rditer/replica_data_iter.go

@@ -372,7 +372,7 @@ func IterateReplicaKeySpans(
 	reader storage.Reader,
 	replicatedOnly bool,
 	replicatedSpansFilter ReplicatedSpansFilter,
-	visitor func(storage.EngineIterator, roachpb.Span, storage.IterKeyType) error,
+	visitor func(storage.EngineIterator, roachpb.Span) error,
 ) error {
 	if !reader.ConsistentIterators() {
 		panic("reader must provide consistent iterators")
@@ -394,28 +394,25 @@ func IterateReplicaKeySpans(
 			UnreplicatedByRangeID: true,
 		})
 	}
-	keyTypes := []storage.IterKeyType{storage.IterKeyTypePointsOnly, storage.IterKeyTypeRangesOnly}
 	for _, span := range spans {
-		for _, keyType := range keyTypes {
-			err := func() error {
-				iter, err := reader.NewEngineIterator(storage.IterOptions{
-					KeyTypes:   keyType,
-					LowerBound: span.Key,
-					UpperBound: span.EndKey,
-				})
-				if err != nil {
-					return err
-				}
-				defer iter.Close()
-				ok, err := iter.SeekEngineKeyGE(storage.EngineKey{Key: span.Key})
-				if err == nil && ok {
-					err = visitor(iter, span, keyType)
-				}
-				return err
-			}()
+		err := func() error {
+			iter, err := reader.NewEngineIterator(storage.IterOptions{
+				KeyTypes:   storage.IterKeyTypePointsAndRanges,
+				LowerBound: span.Key,
+				UpperBound: span.EndKey,
+			})
 			if err != nil {
-				return iterutil.Map(err)
+				return err
 			}
+			defer iter.Close()
+			ok, err := iter.SeekEngineKeyGE(storage.EngineKey{Key: span.Key})
+			if err == nil && ok {
+				err = visitor(iter, span)
+			}
+			return err
+		}()
+		if err != nil {
+			return iterutil.Map(err)
 		}
 	}
 	return nil

pkg/kv/kvserver/rditer/replica_data_iter_test.go

@@ -158,8 +158,9 @@ func verifyIterateReplicaKeySpans(
 		"pretty",
 	})
 
-	require.NoError(t, IterateReplicaKeySpans(desc, readWriter, replicatedOnly, replicatedSpansFilter,
-		func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error {
+	require.NoError(t, IterateReplicaKeySpans(desc, readWriter, replicatedOnly,
+		replicatedSpansFilter,
+		func(iter storage.EngineIterator, span roachpb.Span) error {
 			var err error
 			for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() {
 				// Span should not be empty.
@@ -170,8 +171,8 @@ func verifyIterateReplicaKeySpans(
 				require.True(t, span.ContainsKey(key.Key), "%s not in %s", key, span)
 				require.True(t, key.IsLockTableKey() || key.IsMVCCKey(), "%s neither lock nor MVCC", key)
 
-				switch keyType {
-				case storage.IterKeyTypePointsOnly:
+				hasPoint, hasRange := iter.HasPointAndRange()
+				if hasPoint {
 					var mvccKey storage.MVCCKey
 					if key.IsMVCCKey() {
 						var err error
@@ -197,8 +198,8 @@ func verifyIterateReplicaKeySpans(
 						fmt.Sprintf("%x", key.Version),
 						mvccKey.String(),
 					})
-
-				case storage.IterKeyTypeRangesOnly:
+				}
+				if hasRange && iter.RangeKeyChanged() {
 					bounds, err := iter.EngineRangeBounds()
 					require.NoError(t, err)
 					require.True(t, span.Contains(bounds), "%s not contained in %s", bounds, span)
@@ -218,9 +219,6 @@ func verifyIterateReplicaKeySpans(
 							mvccRangeKey.String(),
 						})
 					}
-
-				default:
-					t.Fatalf("unexpected key type %v", keyType)
 				}
 			}
 			return err
@@ -465,9 +463,11 @@ func TestReplicaDataIteratorGlobalRangeKey(t *testing.T) {
 
 				var actualSpans []roachpb.Span
 				require.NoError(t, IterateReplicaKeySpans(&desc, snapshot, replicatedOnly, ReplicatedSpansAll,
-					func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error {
+					func(iter storage.EngineIterator, span roachpb.Span) error {
 						// We should never see any point keys.
-						require.Equal(t, storage.IterKeyTypeRangesOnly, keyType)
+						hasPoint, hasRange := iter.HasPointAndRange()
+						require.False(t, hasPoint)
+						require.True(t, hasRange)
 
 						// The iterator should already be positioned on the range key, which should
 						// span the entire key span and be the only range key.
@@ -572,7 +572,7 @@ func benchReplicaEngineDataIterator(b *testing.B, numRanges, numKeysPerRange, va
 	for i := 0; i < b.N; i++ {
 		for _, desc := range descs {
 			err := IterateReplicaKeySpans(&desc, snapshot, false /* replicatedOnly */, ReplicatedSpansAll,
-				func(iter storage.EngineIterator, _ roachpb.Span, _ storage.IterKeyType) error {
+				func(iter storage.EngineIterator, _ roachpb.Span) error {
 					var err error
 					for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() {
 						_, _ = iter.UnsafeEngineKey()

pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output

@@ -3,25 +3,25 @@ echo
 +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+
 |              SPAN               |                           KEY HEX                            |  ENDKEY HEX  |            VERSION HEX             |                                       PRETTY                                       |
 +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+
+| /Local/RangeID/1/{r""-s""}      | 016989723a61                                                 | 016989723a78 | 000000000000000109                 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0                                          |
 | /Local/RangeID/1/{r""-s""}      | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001       |              |                                    | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c"                |
 | /Local/RangeID/1/{r""-s""}      | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001       |              |                                    | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b"                |
 | /Local/RangeID/1/{r""-s""}      | 016989726c67632d                                             |              |                                    | /Local/RangeID/1/r/RangeGCThreshold                                                |
 | /Local/RangeID/1/{r""-s""}      | 016989727261736b                                             |              |                                    | /Local/RangeID/1/r/RangeAppliedState                                               |
 | /Local/RangeID/1/{r""-s""}      | 01698972726c6c2d                                             |              |                                    | /Local/RangeID/1/r/RangeLease                                                      |
-| /Local/RangeID/1/{r""-s""}      | 016989723a61                                                 | 016989723a78 | 000000000000000109                 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0                                          |
+| /Local/RangeID/1/{u""-v""}      | 016989753a61                                                 | 016989753a78 | 000000000000000109                 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0                                          |
 | /Local/RangeID/1/{u""-v""}      | 0169897572667462                                             |              |                                    | /Local/RangeID/1/u/RangeTombstone                                                  |
 | /Local/RangeID/1/{u""-v""}      | 0169897572667468                                             |              |                                    | /Local/RangeID/1/u/RaftHardState                                                   |
 | /Local/RangeID/1/{u""-v""}      | 016989757266746c0000000000000001                             |              |                                    | /Local/RangeID/1/u/RaftLog/logIndex:1                                              |
 | /Local/RangeID/1/{u""-v""}      | 016989757266746c0000000000000002                             |              |                                    | /Local/RangeID/1/u/RaftLog/logIndex:2                                              |
 | /Local/RangeID/1/{u""-v""}      | 01698975726c7274                                             |              |                                    | /Local/RangeID/1/u/RangeLastReplicaGCTimestamp                                     |
-| /Local/RangeID/1/{u""-v""}      | 016989753a61                                                 | 016989753a78 | 000000000000000109                 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0                                          |
 | /Local/Range"{a"-b"}            | 016b1261000172647363                                         |              | 0000000000000001                   | /Local/Range"a"/RangeDescriptor/0.000000001,0                                      |
 | /Local/Range"{a"-b"}            | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c         |              |                                    | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c"                 |
 | /Local/Range"{a"-b"}            | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b     |              |                                    | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b"             |
 | /Local/Range"{a"-b"}            | 016b1261ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f |              |                                    | /Local/Range"a\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" |
 | /Local/Lock/Local/Range"{a"-b"} | 017a6b12016b126100ff01726473630001                           |              | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"a"/RangeDescriptor                                                    |
 | /Local/Lock"{a"-b"}             | 017a6b12610001                                               |              | 030ce61c175eb445878c36dcf4062ada4c | "a"                                                                                |
+| {a-b}                           | 61                                                           | 62           | 000000000000000109                 | {a-b}/0.000000001,0                                                                |
 | {a-b}                           | 61                                                           |              | 0000000000000001                   | "a"/0.000000001,0                                                                  |
 | {a-b}                           | 61ffffffff                                                   |              | 0000000000000001                   | "a\xff\xff\xff\xff"/0.000000001,0                                                  |
-| {a-b}                           | 61                                                           | 62           | 000000000000000109                 | {a-b}/0.000000001,0                                                                |
 +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+