Skip to content

Commit 0456569

Browse files
mschochmschoch
authored
Merge pull request #724 from blevesearch/scorch
Scorch
2 parents 53a7575 + 94b0367 commit 0456569

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+15138
-57
lines changed

‎cmd/bleve/cmd/scorch.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright (c) 2017 Couchbase, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package cmd
16+
17+
import (
18+
"github.com/blevesearch/bleve/cmd/bleve/cmd/scorch"
19+
)
20+
21+
// make scorch command-line tool a bleve sub-command
22+
23+
func init() {
24+
RootCmd.AddCommand(scorch.RootCmd)
25+
}

‎cmd/bleve/cmd/scorch/ascii.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright (c) 2017 Couchbase, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package scorch
16+
17+
import (
18+
"fmt"
19+
"strconv"
20+
21+
"github.com/blevesearch/bleve/index/scorch/mergeplan"
22+
"github.com/spf13/cobra"
23+
)
24+
25+
// asciiCmd represents the snapshots command
26+
var asciiCmd = &cobra.Command{
27+
Use: "ascii",
28+
Short: "ascii prints details an ascii representation of the snapshots in the index",
29+
Long: `The ascii command prints an ascii representation of the snapshots in the index.`,
30+
RunE: func(cmd *cobra.Command, args []string) error {
31+
32+
if len(args) < 2 {
33+
return fmt.Errorf("snapshot epoch required")
34+
} else if len(args) < 3 {
35+
snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)
36+
if err != nil {
37+
return err
38+
}
39+
snapshot, err := index.LoadSnapshot(snapshotEpoch)
40+
if err != nil {
41+
return err
42+
}
43+
segments := snapshot.Segments()
44+
var mergePlanSegments []mergeplan.Segment
45+
for _, v := range segments {
46+
mergePlanSegments = append(mergePlanSegments, v)
47+
}
48+
49+
str := mergeplan.ToBarChart(args[1], 25, mergePlanSegments, nil)
50+
fmt.Printf("%s\n", str)
51+
}
52+
53+
return nil
54+
},
55+
}
56+
57+
func init() {
58+
RootCmd.AddCommand(asciiCmd)
59+
}

‎cmd/bleve/cmd/scorch/deleted.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Copyright (c) 2017 Couchbase, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package scorch
16+
17+
import (
18+
"fmt"
19+
"strconv"
20+
21+
"github.com/spf13/cobra"
22+
)
23+
24+
// deletedCmd represents the deleted command
25+
var deletedCmd = &cobra.Command{
26+
Use: "deleted",
27+
Short: "deleted prints the deleted bitmap for segments in the index snapshot",
28+
Long: `The delete command prints the deleted bitmap for segments in the index snapshot.`,
29+
RunE: func(cmd *cobra.Command, args []string) error {
30+
31+
if len(args) < 2 {
32+
return fmt.Errorf("snapshot epoch required")
33+
} else if len(args) < 3 {
34+
snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)
35+
if err != nil {
36+
return err
37+
}
38+
snapshot, err := index.LoadSnapshot(snapshotEpoch)
39+
if err != nil {
40+
return err
41+
}
42+
segments := snapshot.Segments()
43+
for i, segmentSnap := range segments {
44+
deleted := segmentSnap.Deleted()
45+
fmt.Printf("%d %v\n", i, deleted)
46+
}
47+
}
48+
49+
return nil
50+
},
51+
}
52+
53+
func init() {
54+
RootCmd.AddCommand(deletedCmd)
55+
}

‎cmd/bleve/cmd/scorch/info.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright (c) 2017 Couchbase, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package scorch
16+
17+
import (
18+
"fmt"
19+
20+
"github.com/spf13/cobra"
21+
)
22+
23+
// dictCmd represents the dict command
24+
var infoCmd = &cobra.Command{
25+
Use: "info",
26+
Short: "info prints basic info about the index",
27+
Long: `The info command prints basic info about the index.`,
28+
RunE: func(cmd *cobra.Command, args []string) error {
29+
30+
reader, err := index.Reader()
31+
if err != nil {
32+
return err
33+
}
34+
35+
count, err := reader.DocCount()
36+
if err != nil {
37+
return err
38+
}
39+
40+
fmt.Printf("count: %d\n", count)
41+
42+
// var numSnapshots int
43+
// var rootSnapshot uint64
44+
// index.VisitBoltSnapshots(func(snapshotEpoch uint64) error {
45+
// if rootSnapshot == 0 {
46+
// rootSnapshot = snapshotEpoch
47+
// }
48+
// numSnapshots++
49+
// return nil
50+
// })
51+
// fmt.Printf("has %d snapshot(s), root: %d\n", numSnapshots, rootSnapshot)
52+
53+
return nil
54+
},
55+
}
56+
57+
func init() {
58+
RootCmd.AddCommand(infoCmd)
59+
}

‎cmd/bleve/cmd/scorch/internal.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// Copyright (c) 2017 Couchbase, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package scorch
16+
17+
import (
18+
"fmt"
19+
"strconv"
20+
21+
"github.com/spf13/cobra"
22+
)
23+
24+
var ascii bool
25+
26+
// internalCmd represents the snapshots command
27+
var internalCmd = &cobra.Command{
28+
Use: "internal",
29+
Short: "internal prints the internal k/v pairs in a snapshot",
30+
Long: `The internal command prints the internal k/v pairs in a snapshot.`,
31+
RunE: func(cmd *cobra.Command, args []string) error {
32+
33+
if len(args) < 2 {
34+
return fmt.Errorf("snapshot epoch required")
35+
} else if len(args) < 3 {
36+
snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)
37+
if err != nil {
38+
return err
39+
}
40+
snapshot, err := index.LoadSnapshot(snapshotEpoch)
41+
if err != nil {
42+
return err
43+
}
44+
internal := snapshot.Internal()
45+
for k, v := range internal {
46+
if ascii {
47+
fmt.Printf("%s %s\n", k, string(v))
48+
} else {
49+
fmt.Printf("%x %x\n", k, v)
50+
}
51+
}
52+
}
53+
54+
return nil
55+
},
56+
}
57+
58+
func init() {
59+
RootCmd.AddCommand(internalCmd)
60+
internalCmd.Flags().BoolVarP(&ascii, "ascii", "a", false, "print key/value in ascii")
61+
}

‎cmd/bleve/cmd/scorch/root.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright (c) 2017 Couchbase, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package scorch
16+
17+
import (
18+
"fmt"
19+
"os"
20+
21+
"github.com/blevesearch/bleve/index/scorch"
22+
"github.com/spf13/cobra"
23+
)
24+
25+
var index *scorch.Scorch
26+
27+
// RootCmd represents the base command when called without any subcommands
28+
var RootCmd = &cobra.Command{
29+
Use: "scorch",
30+
Short: "command-line tool to interact with a scorch index",
31+
Long: `Scorch is a command-line tool to interact with a scorch index.`,
32+
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
33+
34+
if len(args) < 1 {
35+
return fmt.Errorf("must specify path to scorch index")
36+
}
37+
38+
readOnly := true
39+
config := map[string]interface{}{
40+
"read_only": readOnly,
41+
"path": args[0],
42+
}
43+
44+
idx, err := scorch.NewScorch(scorch.Name, config, nil)
45+
if err != nil {
46+
return err
47+
}
48+
49+
err = idx.Open()
50+
if err != nil {
51+
return fmt.Errorf("error opening: %v", err)
52+
}
53+
54+
index = idx.(*scorch.Scorch)
55+
56+
return nil
57+
},
58+
PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
59+
return nil
60+
},
61+
}
62+
63+
// Execute adds all child commands to the root command sets flags appropriately.
64+
// This is called by main.main(). It only needs to happen once to the rootCmd.
65+
func Execute() {
66+
if err := RootCmd.Execute(); err != nil {
67+
fmt.Println(err)
68+
os.Exit(-1)
69+
}
70+
}

‎cmd/bleve/cmd/scorch/snapshot.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// Copyright (c) 2017 Couchbase, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package scorch
16+
17+
import (
18+
"fmt"
19+
"strconv"
20+
21+
"github.com/blevesearch/bleve/index/scorch/segment/zap"
22+
"github.com/spf13/cobra"
23+
)
24+
25+
// snapshotCmd represents the snapshot command
26+
var snapshotCmd = &cobra.Command{
27+
Use: "snapshot",
28+
Short: "info prints details about the snapshots in the index",
29+
Long: `The snapshot command prints details about the snapshots in the index.`,
30+
RunE: func(cmd *cobra.Command, args []string) error {
31+
32+
if len(args) < 2 {
33+
snapshotEpochs, err := index.RootBoltSnapshotEpochs()
34+
if err != nil {
35+
return err
36+
}
37+
for _, snapshotEpoch := range snapshotEpochs {
38+
fmt.Printf("%d\n", snapshotEpoch)
39+
}
40+
} else if len(args) < 3 {
41+
snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)
42+
if err != nil {
43+
return err
44+
}
45+
snapshot, err := index.LoadSnapshot(snapshotEpoch)
46+
if err != nil {
47+
return err
48+
}
49+
segments := snapshot.Segments()
50+
for i, segmentSnap := range segments {
51+
segment := segmentSnap.Segment()
52+
if segment, ok := segment.(*zap.Segment); ok {
53+
fmt.Printf("%d %s\n", i, segment.Path())
54+
}
55+
}
56+
}
57+
58+
return nil
59+
},
60+
}
61+
62+
func init() {
63+
RootCmd.AddCommand(snapshotCmd)
64+
}

0 commit comments

Comments
 (0)