Allow embedded config to come from an external file

accessors/zip/me.go

@@ -2,10 +2,10 @@ package zip
 
 import (
 	"io"
-	"os"
 
 	"github.com/go-errors/errors"
 	"www.velocidex.com/golang/velociraptor/accessors"
+	"www.velocidex.com/golang/velociraptor/config"
 	"www.velocidex.com/golang/velociraptor/third_party/zip"
 	"www.velocidex.com/golang/vfilter"
 )
@@ -19,10 +19,7 @@ type MEFileSystemAccessor struct {
 func (self *MEFileSystemAccessor) GetZipFile(file_path *accessors.OSPath) (
 	*ZipFileCache, error) {
 
-	me, err := os.Executable()
-	if err != nil {
-		return nil, err
-	}
+	me := config.EmbeddedFile
 
 	mu.Lock()
 	zip_file_cache, pres := self.fd_cache[me]

artifacts/definitions/Server/Internal/ToolDependencies.yaml

@@ -19,8 +19,13 @@ tools:
     serve_locally: true
     version: 0.7.0-rc1
 
-  - name: VelociraptorDarwin
-    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-rc1-darwin-amd64
+  # On MacOS we can not embed the config in the binary so we use a
+  # shell script stub instead. See
+  # https://github.com/Velocidex/velociraptor/issues/2898
+
+  # A Generic collector to be used with the --embedded_config flag.
+  - name: VelociraptorCollector
+    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-collector
     serve_locally: true
     version: 0.7.0-rc1
 

artifacts/definitions/Server/Utils/CreateCollector.yaml

@@ -370,8 +370,10 @@ sources:
        a={ SELECT "VelociraptorWindows" AS Type FROM scope() WHERE OS = "Windows"},
        b={ SELECT "VelociraptorWindows_x86" AS Type FROM scope() WHERE OS = "Windows_x86"},
        c={ SELECT "VelociraptorLinux" AS Type FROM scope() WHERE OS = "Linux"},
-       d={ SELECT "VelociraptorDarwin" AS Type FROM scope() WHERE OS = "MacOS"},
-       e={ SELECT "" AS Type FROM scope()
+       d={ SELECT "VelociraptorCollector" AS Type FROM scope() WHERE OS = "MacOS"},
+       e={ SELECT "VelociraptorCollector" AS Type FROM scope() WHERE OS = "MacOSArm"},
+       f={ SELECT "VelociraptorCollector" AS Type FROM scope() WHERE OS = "Generic"},
+       g={ SELECT "" AS Type FROM scope()
            WHERE NOT log(message="Unknown target type " + OS) }
       )
 

bin/deaddisk_test.go

@@ -30,7 +30,7 @@ func (self *CollectorTestSuite) TestDeaddisk() {
 	cmd := exec.Command(self.binary, "deaddisk", "-v",
 		"--add_windows_directory", windows_dir, remapping_path)
 	out, err := cmd.CombinedOutput()
-	require.NoError(t, err)
+	require.NoError(t, err, string(out))
 
 	assert.Contains(t, string(out), `Adding windows mounted directory at`)
 

bin/main.go

@@ -43,6 +43,9 @@ var (
 
 	config_path = app.Flag("config", "The configuration file.").
 			Short('c').String()
+
+	embedded_config_path = app.Flag("embedded_config", "Extract the embedded configuration from this file.").String()
+
 	api_config_path = app.Flag("api_config", "The API configuration file.").
 			Short('a').String()
 
@@ -129,7 +132,7 @@ func main() {
 	pre, post := splitArgs(args)
 	if len(pre) == 0 {
 		config_obj, err := new(config.Loader).WithVerbose(*verbose_flag).
-			WithEmbedded().LoadAndValidate()
+			WithEmbedded(*embedded_config_path).LoadAndValidate()
 		if err == nil && config_obj.Autoexec != nil && config_obj.Autoexec.Argv != nil {
 			args = nil
 			for _, arg := range config_obj.Autoexec.Argv {
@@ -163,7 +166,7 @@ func main() {
 		WithCustomValidator("Validator maybe_unlock_api_config",
 			maybe_unlock_api_config).
 		WithFileLoader(*config_path).
-		WithEmbedded().
+		WithEmbedded(*embedded_config_path).
 		WithEnvLoader("VELOCIRAPTOR_CONFIG").
 		WithConfigMutator("Mutator mergeFlagConfig",
 			func(config_obj *config_proto.Config) error {
@@ -210,7 +213,7 @@ func makeDefaultConfigLoader() *config.Loader {
 		WithVerbose(*verbose_flag).
 		WithTempdir(*tempdir_flag).
 		WithFileLoader(*config_path).
-		WithEmbedded().
+		WithEmbedded(*embedded_config_path).
 		WithEnvLoader("VELOCIRAPTOR_CONFIG").
 		WithConfigMutator("Mutator mergeFlagConfig",
 			func(config_obj *config_proto.Config) error {
@@ -231,12 +234,21 @@ func makeDefaultConfigLoader() *config.Loader {
 // Split the command line into args before the -- and after the --
 func splitArgs(args []string) (pre, post []string) {
 	seen := false
-	for _, arg := range args {
+	for idx := 0; idx < len(args); idx++ {
+		arg := args[idx]
+
+		// Separate the args into pre and post args. Post args will be
+		// added to the autoexec command line while still triggering
+		// the autoexec condition.
 		if arg == "--" {
 			seen = true
 			continue
 		}
 
+		if arg == "--embedded_config" && idx < len(args) {
+			embedded_config_path = &args[idx+1]
+		}
+
 		if seen {
 			post = append(post, arg)
 		} else {

bin/panic.go

@@ -16,7 +16,7 @@ func writeLogOnPanic() error {
 	// Figure out the log directory.
 	config_obj, err := new(config.Loader).
 		WithFileLoader(*config_path).
-		WithEmbedded().
+		WithEmbedded(*embedded_config_path).
 		WithEnvLoader("VELOCIRAPTOR_CONFIG").
 		LoadAndValidate()
 	if err != nil {

bin/repack.go

@@ -50,9 +50,9 @@ var (
 		"config_file", "The filename to write into the binary.").
 		Required().File()
 
-	repack_command_append = repack_command.Flag(
-		"append", "If provided we append the file to the output binary.").
-		File()
+	repack_command_binaries = repack_command.Flag(
+		"binaries", "The list of tool names to append to the binary.").
+		Strings()
 
 	repack_command_output = repack_command.Arg(
 		"output", "The filename to write the repacked binary.").
@@ -91,10 +91,26 @@ func doRepack() error {
 	}
 
 	config_obj := &config_proto.Config{}
+	if isConfigSpecified(os.Args) {
+		config_obj, err = APIConfigLoader.WithNullLoader().
+			LoadAndValidate()
+		if err != nil {
+			return err
+		}
+		config_obj.Services = services.GenericToolServices()
+		if config_obj.Datastore != nil && config_obj.Datastore.Location != "" {
+			config_obj.Services.IndexServer = true
+			config_obj.Services.ClientInfo = true
+			config_obj.Services.Label = true
+		}
+
+	} else {
+		config_obj.Services = services.GenericToolServices()
+	}
+
 	ctx, cancel := install_sig_handler()
 	defer cancel()
 
-	config_obj.Services = services.GenericToolServices()
 	sm, err := startup.StartToolServices(ctx, config_obj)
 	defer sm.Close()
 
@@ -116,11 +132,12 @@ func doRepack() error {
 		Env: ordereddict.NewDict().
 			Set("ConfigData", config_data).
 			Set("Exe", executable).
+			Set("Binaries", *repack_command_binaries).
 			Set("UploadName", filepath.Base(output_path)),
 	}
 
 	query := `
-       SELECT repack(exe=Exe, accessor="file",
+       SELECT repack(exe=Exe, accessor="file", binaries=Binaries,
           config=ConfigData, upload_name=UploadName) AS RepackInfo
        FROM scope()
 `

bin/utils.go

@@ -156,5 +156,13 @@ func install_sig_handler() (context.Context, context.CancelFunc) {
 	}()
 
 	return ctx, cancel
+}
 
+func isConfigSpecified(argv []string) bool {
+	for _, a := range argv {
+		if a == "--config" {
+			return true
+		}
+	}
+	return false
 }

config/loader.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/user"
+	"regexp"
 	"runtime"
 
 	"github.com/Velocidex/yaml/v2"
@@ -17,6 +18,15 @@ import (
 	"www.velocidex.com/golang/velociraptor/utils"
 )
 
+var (
+	noEmbeddedConfig = errors.New(
+		"No embedded config - you can pack one with the `config repack` command")
+
+	embedded_re = regexp.MustCompile(`#{3}<Begin Embedded Config>\r?\n`)
+
+	EmbeddedFile = ""
+)
+
 // A hard error causes the loader to stop immediately.
 type HardError struct {
 	Err error
@@ -323,14 +333,49 @@ func (self *Loader) WithEnvLiteralLoader(env_var string) *Loader {
 	return self
 }
 
-func (self *Loader) WithEmbedded() *Loader {
+func (self *Loader) WithEmbedded(embedded_file string) *Loader {
 	self = self.Copy()
 	self.loaders = append(self.loaders, loaderFunction{
 		name: "WithEmbedded",
 		loader_func: func(self *Loader) (*config_proto.Config, error) {
-			result, err := read_embedded_config()
-			if err == nil {
+			if embedded_file == "" {
+				result, err := read_embedded_config()
+				if err != nil {
+					return nil, err
+				}
+
 				self.Log("Loaded embedded config")
+
+				EmbeddedFile, err = os.Executable()
+				return result, err
+			}
+
+			// Ensure the "me" accessor uses this file for embedded zip.
+			EmbeddedFile = embedded_file
+
+			fd, err := os.Open(embedded_file)
+			if err != nil {
+				return nil, err
+			}
+
+			buf := make([]byte, len(FileConfigDefaultYaml)+1024)
+			n, err := fd.Read(buf)
+			if err != nil {
+				return nil, err
+			}
+
+			buf = buf[:n]
+
+			// Find the embedded marker in the buffer.
+			match := embedded_re.FindIndex(buf)
+			if match == nil {
+				return nil, noEmbeddedConfig
+			}
+
+			embedded_string := buf[match[0]:]
+			result, err := decode_embedded_config(embedded_string)
+			if err == nil {
+				self.Log("Loaded embedded config from %v", embedded_file)
 			}
 			return result, err
 		}})
@@ -487,22 +532,29 @@ func (self *Loader) LoadAndValidate() (*config_proto.Config, error) {
 }
 
 func read_embedded_config() (*config_proto.Config, error) {
+	return decode_embedded_config(FileConfigDefaultYaml)
+}
+
+func decode_embedded_config(encoded_string []byte) (*config_proto.Config, error) {
 	// Get the first line which is never disturbed
-	idx := bytes.IndexByte(FileConfigDefaultYaml, '\n')
+	idx := bytes.IndexByte(encoded_string, '\n')
+
+	if len(encoded_string) < idx+10 {
+		return nil, noEmbeddedConfig
+	}
 
 	// If the following line still starts with # then the file is not
 	// repacked - the repacker will replace all further data with the
 	// compressed string.
-	if FileConfigDefaultYaml[idx+1] == '#' {
-		return nil, errors.New(
-			"No embedded config - you can pack one with the `config repack` command")
+	if encoded_string[idx+1] == '#' {
+		return nil, noEmbeddedConfig
 	}
 
 	// Decompress the rest of the data - note that zlib will ignore
 	// any padding anyway because the zlib header already contains the
 	// length of the compressed data so it is safe to just feed it the
 	// whole string here.
-	r, err := zlib.NewReader(bytes.NewReader(FileConfigDefaultYaml[idx+1:]))
+	r, err := zlib.NewReader(bytes.NewReader(encoded_string[idx+1:]))
 	if err != nil {
 		return nil, err
 	}