Allow embedded config to come from an external file (#2899)

On MacOS we can not modify the binary after build due to hashes and
verifications. This PR allows the embedded config to be written to a
stub file (shell script) which can be used later to bootstrap the
offline collector.

This approach is a bit more complicated but will also be useful on
Windows where the repack operation invalidates the signature. We can
still use this same approach on Windows too without needing to resign
the binary.

accessors/zip/me.go

@@ -2,10 +2,10 @@ package zip
 
 import (
 	"io"
-	"os"
 
 	"github.com/go-errors/errors"
 	"www.velocidex.com/golang/velociraptor/accessors"
+	"www.velocidex.com/golang/velociraptor/config"
 	"www.velocidex.com/golang/velociraptor/third_party/zip"
 	"www.velocidex.com/golang/vfilter"
 )
@@ -19,10 +19,7 @@ type MEFileSystemAccessor struct {
 func (self *MEFileSystemAccessor) GetZipFile(file_path *accessors.OSPath) (
 	*ZipFileCache, error) {
 
-	me, err := os.Executable()
-	if err != nil {
-		return nil, err
-	}
+	me := config.EmbeddedFile
 
 	mu.Lock()
 	zip_file_cache, pres := self.fd_cache[me]

artifacts/definitions/Server/Internal/ToolDependencies.yaml

@@ -19,8 +19,13 @@ tools:
     serve_locally: true
     version: 0.7.0-rc1
 
-  - name: VelociraptorDarwin
-    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-rc1-darwin-amd64
+  # On MacOS we can not embed the config in the binary so we use a
+  # shell script stub instead. See
+  # https://github.com/Velocidex/velociraptor/issues/2898
+
+  # A Generic collector to be used with the --embedded_config flag.
+  - name: VelociraptorCollector
+    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-collector
     serve_locally: true
     version: 0.7.0-rc1
 

artifacts/definitions/Server/Utils/CreateCollector.yaml

@@ -370,8 +370,10 @@ sources:
        a={ SELECT "VelociraptorWindows" AS Type FROM scope() WHERE OS = "Windows"},
        b={ SELECT "VelociraptorWindows_x86" AS Type FROM scope() WHERE OS = "Windows_x86"},
        c={ SELECT "VelociraptorLinux" AS Type FROM scope() WHERE OS = "Linux"},
-       d={ SELECT "VelociraptorDarwin" AS Type FROM scope() WHERE OS = "MacOS"},
-       e={ SELECT "" AS Type FROM scope()
+       d={ SELECT "VelociraptorCollector" AS Type FROM scope() WHERE OS = "MacOS"},
+       e={ SELECT "VelociraptorCollector" AS Type FROM scope() WHERE OS = "MacOSArm"},
+       f={ SELECT "VelociraptorCollector" AS Type FROM scope() WHERE OS = "Generic"},
+       g={ SELECT "" AS Type FROM scope()
            WHERE NOT log(message="Unknown target type " + OS) }
       )
 

bin/deaddisk_test.go

@@ -30,7 +30,7 @@ func (self *CollectorTestSuite) TestDeaddisk() {
 	cmd := exec.Command(self.binary, "deaddisk", "-v",
 		"--add_windows_directory", windows_dir, remapping_path)
 	out, err := cmd.CombinedOutput()
-	require.NoError(t, err)
+	require.NoError(t, err, string(out))
 
 	assert.Contains(t, string(out), `Adding windows mounted directory at`)
 

bin/main.go

@@ -43,6 +43,9 @@ var (
 
 	config_path = app.Flag("config", "The configuration file.").
 			Short('c').String()
+
+	embedded_config_path = app.Flag("embedded_config", "Extract the embedded configuration from this file.").String()
+
 	api_config_path = app.Flag("api_config", "The API configuration file.").
 			Short('a').String()
 
@@ -129,7 +132,7 @@ func main() {
 	pre, post := splitArgs(args)
 	if len(pre) == 0 {
 		config_obj, err := new(config.Loader).WithVerbose(*verbose_flag).
-			WithEmbedded().LoadAndValidate()
+			WithEmbedded(*embedded_config_path).LoadAndValidate()
 		if err == nil && config_obj.Autoexec != nil && config_obj.Autoexec.Argv != nil {
 			args = nil
 			for _, arg := range config_obj.Autoexec.Argv {
@@ -163,7 +166,7 @@ func main() {
 		WithCustomValidator("Validator maybe_unlock_api_config",
 			maybe_unlock_api_config).
 		WithFileLoader(*config_path).
-		WithEmbedded().
+		WithEmbedded(*embedded_config_path).
 		WithEnvLoader("VELOCIRAPTOR_CONFIG").
 		WithConfigMutator("Mutator mergeFlagConfig",
 			func(config_obj *config_proto.Config) error {
@@ -210,7 +213,7 @@ func makeDefaultConfigLoader() *config.Loader {
 		WithVerbose(*verbose_flag).
 		WithTempdir(*tempdir_flag).
 		WithFileLoader(*config_path).
-		WithEmbedded().
+		WithEmbedded(*embedded_config_path).
 		WithEnvLoader("VELOCIRAPTOR_CONFIG").
 		WithConfigMutator("Mutator mergeFlagConfig",
 			func(config_obj *config_proto.Config) error {
@@ -231,12 +234,21 @@ func makeDefaultConfigLoader() *config.Loader {
 // Split the command line into args before the -- and after the --
 func splitArgs(args []string) (pre, post []string) {
 	seen := false
-	for _, arg := range args {
+	for idx := 0; idx < len(args); idx++ {
+		arg := args[idx]
+
+		// Separate the args into pre and post args. Post args will be
+		// added to the autoexec command line while still triggering
+		// the autoexec condition.
 		if arg == "--" {
 			seen = true
 			continue
 		}
 
+		if arg == "--embedded_config" && idx < len(args) {
+			embedded_config_path = &args[idx+1]
+		}
+
 		if seen {
 			post = append(post, arg)
 		} else {

bin/panic.go

@@ -16,7 +16,7 @@ func writeLogOnPanic() error {
 	// Figure out the log directory.
 	config_obj, err := new(config.Loader).
 		WithFileLoader(*config_path).
-		WithEmbedded().
+		WithEmbedded(*embedded_config_path).
 		WithEnvLoader("VELOCIRAPTOR_CONFIG").
 		LoadAndValidate()
 	if err != nil {

bin/repack.go

@@ -50,9 +50,9 @@ var (
 		"config_file", "The filename to write into the binary.").
 		Required().File()
 
-	repack_command_append = repack_command.Flag(
-		"append", "If provided we append the file to the output binary.").
-		File()
+	repack_command_binaries = repack_command.Flag(
+		"binaries", "The list of tool names to append to the binary.").
+		Strings()
 
 	repack_command_output = repack_command.Arg(
 		"output", "The filename to write the repacked binary.").
@@ -91,10 +91,26 @@ func doRepack() error {
 	}
 
 	config_obj := &config_proto.Config{}
+	if isConfigSpecified(os.Args) {
+		config_obj, err = APIConfigLoader.WithNullLoader().
+			LoadAndValidate()
+		if err != nil {
+			return err
+		}
+		config_obj.Services = services.GenericToolServices()
+		if config_obj.Datastore != nil && config_obj.Datastore.Location != "" {
+			config_obj.Services.IndexServer = true
+			config_obj.Services.ClientInfo = true
+			config_obj.Services.Label = true
+		}
+
+	} else {
+		config_obj.Services = services.GenericToolServices()
+	}
+
 	ctx, cancel := install_sig_handler()
 	defer cancel()
 
-	config_obj.Services = services.GenericToolServices()
 	sm, err := startup.StartToolServices(ctx, config_obj)
 	defer sm.Close()
 
@@ -116,11 +132,12 @@ func doRepack() error {
 		Env: ordereddict.NewDict().
 			Set("ConfigData", config_data).
 			Set("Exe", executable).
+			Set("Binaries", *repack_command_binaries).
 			Set("UploadName", filepath.Base(output_path)),
 	}
 
 	query := `
-       SELECT repack(exe=Exe, accessor="file",
+       SELECT repack(exe=Exe, accessor="file", binaries=Binaries,
           config=ConfigData, upload_name=UploadName) AS RepackInfo
        FROM scope()
 `

bin/utils.go

@@ -156,5 +156,13 @@ func install_sig_handler() (context.Context, context.CancelFunc) {
 	}()
 
 	return ctx, cancel
+}
 
+func isConfigSpecified(argv []string) bool {
+	for _, a := range argv {
+		if a == "--config" {
+			return true
+		}
+	}
+	return false
 }

config/loader.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/user"
+	"regexp"
 	"runtime"
 
 	"github.com/Velocidex/yaml/v2"
@@ -17,6 +18,15 @@ import (
 	"www.velocidex.com/golang/velociraptor/utils"
 )
 
+var (
+	noEmbeddedConfig = errors.New(
+		"No embedded config - you can pack one with the `config repack` command")
+
+	embedded_re = regexp.MustCompile(`#{3}<Begin Embedded Config>\r?\n`)
+
+	EmbeddedFile = ""
+)
+
 // A hard error causes the loader to stop immediately.
 type HardError struct {
 	Err error
@@ -323,14 +333,49 @@ func (self *Loader) WithEnvLiteralLoader(env_var string) *Loader {
 	return self
 }
 
-func (self *Loader) WithEmbedded() *Loader {
+func (self *Loader) WithEmbedded(embedded_file string) *Loader {
 	self = self.Copy()
 	self.loaders = append(self.loaders, loaderFunction{
 		name: "WithEmbedded",
 		loader_func: func(self *Loader) (*config_proto.Config, error) {
-			result, err := read_embedded_config()
-			if err == nil {
+			if embedded_file == "" {
+				result, err := read_embedded_config()
+				if err != nil {
+					return nil, err
+				}
+
 				self.Log("Loaded embedded config")
+
+				EmbeddedFile, err = os.Executable()
+				return result, err
+			}
+
+			// Ensure the "me" accessor uses this file for embedded zip.
+			EmbeddedFile = embedded_file
+
+			fd, err := os.Open(embedded_file)
+			if err != nil {
+				return nil, err
+			}
+
+			buf := make([]byte, len(FileConfigDefaultYaml)+1024)
+			n, err := fd.Read(buf)
+			if err != nil {
+				return nil, err
+			}
+
+			buf = buf[:n]
+
+			// Find the embedded marker in the buffer.
+			match := embedded_re.FindIndex(buf)
+			if match == nil {
+				return nil, noEmbeddedConfig
+			}
+
+			embedded_string := buf[match[0]:]
+			result, err := decode_embedded_config(embedded_string)
+			if err == nil {
+				self.Log("Loaded embedded config from %v", embedded_file)
 			}
 			return result, err
 		}})
@@ -487,22 +532,29 @@ func (self *Loader) LoadAndValidate() (*config_proto.Config, error) {
 }
 
 func read_embedded_config() (*config_proto.Config, error) {
+	return decode_embedded_config(FileConfigDefaultYaml)
+}
+
+func decode_embedded_config(encoded_string []byte) (*config_proto.Config, error) {
 	// Get the first line which is never disturbed
-	idx := bytes.IndexByte(FileConfigDefaultYaml, '\n')
+	idx := bytes.IndexByte(encoded_string, '\n')
+
+	if len(encoded_string) < idx+10 {
+		return nil, noEmbeddedConfig
+	}
 
 	// If the following line still starts with # then the file is not
 	// repacked - the repacker will replace all further data with the
 	// compressed string.
-	if FileConfigDefaultYaml[idx+1] == '#' {
-		return nil, errors.New(
-			"No embedded config - you can pack one with the `config repack` command")
+	if encoded_string[idx+1] == '#' {
+		return nil, noEmbeddedConfig
 	}
 
 	// Decompress the rest of the data - note that zlib will ignore
 	// any padding anyway because the zlib header already contains the
 	// length of the compressed data so it is safe to just feed it the
 	// whole string here.
-	r, err := zlib.NewReader(bytes.NewReader(FileConfigDefaultYaml[idx+1:]))
+	r, err := zlib.NewReader(bytes.NewReader(encoded_string[idx+1:]))
 	if err != nil {
 		return nil, err
 	}