Bugfix: accessors should provide their underlying file (Velocidex#2893)

We sometimes need to know the path to the underlying file on the
filesystem if possible. This is used in cases when we need to delegate
to an external library which expects a filesystem path.

Previously the code assumed that when the accessor was "file" or "auto"
then the underlying path can be obtained from the filename. This works
well in local trigage mode but fails when remapping - in that case the
actual accessor called "file" may be a completely different remapped
accessor and it is not appropriate to use its filename as an underlying
API file.

This would cause issues with e.g. the yara plugin, sqlite and leveldb
plugins.

This PR introduces a new interface which allows the accessor to provide
the raw API accessible path if possible. For plugins that need to work
with real files, this path also creates a local copy if needed.

Fixes: Velocidex#2870

Author: scudette

accessors/api.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/Velocidex/ordereddict"
+	errors "github.com/go-errors/errors"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/utils"
 	"www.velocidex.com/golang/vfilter"
@@ -320,3 +321,31 @@ type FileSystemAccessor interface {
 	LstatWithOSPath(path *OSPath) (FileInfo, error)
 	New(scope vfilter.Scope) (FileSystemAccessor, error)
 }
+
+// Some filesystems can attempt to retrieve the underlying file. If
+// this interface exists on the accessor **and** the
+// GetUnderlyingAPIFilename() call succeeds, then it should be
+// possible to directly access the returned filename using the OS
+// APIs.
+type RawFileAPIAccessor interface {
+	GetUnderlyingAPIFilename(path *OSPath) (string, error)
+}
+
+var (
+	NotRawFileSystem = errors.New("NotRawFileSystem")
+)
+
+func GetUnderlyingAPIFilename(accessor string,
+	scope vfilter.Scope, path *OSPath) (string, error) {
+	accessor_obj, err := GetAccessor(accessor, scope)
+	if err != nil {
+		return "", err
+	}
+
+	raw_accessor, ok := accessor_obj.(RawFileAPIAccessor)
+	if !ok {
+		return "", NotRawFileSystem
+	}
+
+	return raw_accessor.GetUnderlyingAPIFilename(path)
+}

accessors/file/accessor_common.go

@@ -293,6 +293,11 @@ func (self OSFileSystemAccessor) ReadDir(dir string) ([]accessors.FileInfo, erro
 	return self.ReadDirWithOSPath(full_path)
 }
 
+func (self *OSFileSystemAccessor) GetUnderlyingAPIFilename(
+	full_path *accessors.OSPath) (string, error) {
+	return full_path.PathSpec().Path, nil
+}
+
 func (self OSFileSystemAccessor) ReadDirWithOSPath(
 	full_path *accessors.OSPath) ([]accessors.FileInfo, error) {
 	dir := full_path.PathSpec().Path

accessors/file/auto_windows.go

@@ -135,6 +135,11 @@ func (self AutoFilesystemAccessor) New(scope vfilter.Scope) (accessors.FileSyste
 	}, nil
 }
 
+func (self *AutoFilesystemAccessor) GetUnderlyingAPIFilename(
+	full_path *accessors.OSPath) (string, error) {
+	return full_path.PathSpec().Path, nil
+}
+
 func (self *AutoFilesystemAccessor) ReadDirWithOSPath(
 	path *accessors.OSPath) ([]accessors.FileInfo, error) {
 	result, err := self.file_delegate.ReadDirWithOSPath(path)

accessors/file/os_windows.go

@@ -151,6 +151,11 @@ func (self OSFileSystemAccessor) New(scope vfilter.Scope) (
 	return result, nil
 }
 
+func (self *OSFileSystemAccessor) GetUnderlyingAPIFilename(
+	full_path *accessors.OSPath) (string, error) {
+	return full_path.PathSpec().Path, nil
+}
+
 func discoverDriveLetters() ([]accessors.FileInfo, error) {
 	result := []accessors.FileInfo{}
 

artifacts/testdata/server/testcases/remapping.out.yaml

@@ -171,6 +171,7 @@ LET _ <= remap(config=format(format=RemappingTemplate, args=[ srcDir+'/artifacts
 ]SELECT * FROM parse_ntfs_i30( accessor='ntfs', device='c:/$MFT', inode="41-144-1")[
  {
   "MFTId": "45",
+  "SequenceNumber": 0,
   "Mtime": "2018-09-24T07:55:44.4592119Z",
   "Atime": "2022-03-18T04:09:07.2885951Z",
   "Ctime": "2018-09-24T07:55:20.6489276Z",

go.mod

@@ -97,7 +97,7 @@ require (
 	howett.net/plist v1.0.0
 	www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e
 	www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671
-	www.velocidex.com/golang/go-ntfs v0.1.2-0.20230728152253-4d399c766ed6
+	www.velocidex.com/golang/go-ntfs v0.1.2-0.20230815140127-6a3dd72bfbf1
 	www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3
 	www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a
 	www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe

go.sum

@@ -1231,8 +1231,8 @@ www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e h1:AhcXPgNKhJ
 www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e/go.mod h1:ykEQ7AUF9AL+mfCefDmLwmZOnU2So6wM3qKM8xdsHhU=
 www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671 h1:pfvo7NFo0eJj6Zr7d+4vMx/Zr2JriMMPEWRHUf1YjUw=
 www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671/go.mod h1:qnzHyB9yD2khtYO+wf3ck9FQxX3wFhXeJHFBnuUIZcc=
-www.velocidex.com/golang/go-ntfs v0.1.2-0.20230728152253-4d399c766ed6 h1:CQTXpiMZ01PJIvpelSzpWJlZEUoQM831YgHEVdaZic4=
-www.velocidex.com/golang/go-ntfs v0.1.2-0.20230728152253-4d399c766ed6/go.mod h1:itvbHQcnLdTVIDY6fI3lR0zeBwXwBYBdUFtswE0x1vc=
+www.velocidex.com/golang/go-ntfs v0.1.2-0.20230815140127-6a3dd72bfbf1 h1:6NMITYv1pi4tzmDcqB/enNUXKmS8dnTb72HBghqhnAM=
+www.velocidex.com/golang/go-ntfs v0.1.2-0.20230815140127-6a3dd72bfbf1/go.mod h1:itvbHQcnLdTVIDY6fI3lR0zeBwXwBYBdUFtswE0x1vc=
 www.velocidex.com/golang/go-pe v0.1.1-0.20220107093716-e91743c801de/go.mod h1:j9Xy8Z9wxzY2SCB8CqDkkoSzy+eUwevnOrRm/XM2q/A=
 www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3 h1:W394TEIFuHFxHY8mzTJPHI5v+M+NLKEHmHn7KY/VpEM=
 www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3/go.mod h1:agYwYzeeytVtdwkRrvxZAjgIA8SCeM/Tg7Ym2/jBwmA=

vql/common/yara.go

@@ -146,23 +146,32 @@ func (self YaraScanPlugin) Call(
 			}
 			matcher.filename = filename
 
-			// If accessor is not specified we call yara's
-			// ScanFile API which mmaps the entire file
-			// into memory avoiding the need for
-			// buffering.
-			if arg.Accessor == "" || arg.Accessor == "file" {
-				err := matcher.scanFile(ctx, output_chan)
+			accessor, err := accessors.GetAccessor(arg.Accessor, scope)
+			if err != nil {
+				scope.Log("yara: %v", err)
+				return
+			}
+
+			// As an optimization, we try to call yara's ScanFile API
+			// which mmaps the entire file into memory avoiding the
+			// need for buffering.
+			raw_accessor, ok := accessor.(accessors.RawFileAPIAccessor)
+			if ok {
+				underlying_file, err := raw_accessor.GetUnderlyingAPIFilename(filename)
 				if err == nil {
-					continue
-				} else {
-					scope.Log("Directly scanning file %v failed, will use accessor",
-						filename.String())
+					err := matcher.scanFile(ctx, underlying_file, output_chan)
+					if err == nil {
+						continue
+					} else {
+						scope.Log("Directly scanning file %v failed, will use accessor",
+							filename.String())
+					}
 				}
 			}
 
 			// If scanning with the file api failed above
 			// we fall back to accessor scanning.
-			matcher.scanFileByAccessor(ctx, arg.Accessor,
+			matcher.scanFileByAccessor(ctx, arg.Accessor, accessor,
 				arg.Blocksize, arg.Start, arg.End, output_chan)
 		}
 	}()
@@ -234,18 +243,13 @@ func getYaraRules(key, namespace, rules string,
 func (self *scanReporter) scanFileByAccessor(
 	ctx context.Context,
 	accessor_name string,
+	accessor accessors.FileSystemAccessor,
 	blocksize uint64,
 	start, end uint64,
 	output_chan chan vfilter.Row) {
 
 	defer utils.CheckForPanic("Panic in scanFileByAccessor")
 
-	accessor, err := accessors.GetAccessor(accessor_name, self.scope)
-	if err != nil {
-		self.scope.Log("yara: %v", err)
-		return
-	}
-
 	// Open the file with the accessor
 	f, err := accessor.OpenWithOSPath(self.filename)
 	if err != nil {
@@ -354,9 +358,11 @@ func (self *scanReporter) scanRange(start, end uint64, f accessors.ReadSeekClose
 // filename to libyara directly for faster scanning using mmap. This
 // also ensures that all yara features (like the PE plugin) work.
 func (self *scanReporter) scanFile(
-	ctx context.Context, output_chan chan vfilter.Row) error {
+	ctx context.Context,
+	underlying_file string,
+	output_chan chan vfilter.Row) error {
 
-	fd, err := os.Open(self.filename.String())
+	fd, err := os.Open(underlying_file)
 	if err != nil {
 		return err
 	}
@@ -377,7 +383,7 @@ func (self *scanReporter) scanFile(
 	err = scanner.SetCallback(self).
 		SetTimeout(10 * time.Second).
 		SetFlags(self.yara_flag).
-		ScanFile(self.filename.String())
+		ScanFile(underlying_file)
 	if err != nil {
 		return err
 	}

vql/parsers/csv/csv.go

@@ -224,7 +224,7 @@ func (self _WatchCSVPlugin) Info(scope vfilter.Scope, type_map *vfilter.TypeMap)
 }
 
 type WriteCSVPluginArgs struct {
-	Filename string              `vfilter:"required,field=filename,doc=CSV files to open"`
+	Filename *accessors.OSPath   `vfilter:"required,field=filename,doc=CSV files to open"`
 	Accessor string              `vfilter:"optional,field=accessor,doc=The accessor to use"`
 	Query    vfilter.StoredQuery `vfilter:"required,field=query,doc=query to write into the file."`
 }
@@ -257,7 +257,14 @@ func (self WriteCSVPlugin) Call(
 				return
 			}
 
-			file, err := os.OpenFile(arg.Filename,
+			underlying_file, err := accessors.GetUnderlyingAPIFilename(
+				arg.Accessor, scope, arg.Filename)
+			if err != nil {
+				scope.Log("write_csv: %s", err)
+				return
+			}
+
+			file, err := os.OpenFile(underlying_file,
 				os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0700)
 			if err != nil {
 				scope.Log("write_csv: Unable to open file %s: %s",

vql/parsers/json.go

@@ -485,7 +485,7 @@ func (self _IndexAssociativeProtocol) GetMembers(
 }
 
 type WriteJSONPluginArgs struct {
-	Filename string              `vfilter:"required,field=filename,doc=CSV files to open"`
+	Filename *accessors.OSPath   `vfilter:"required,field=filename,doc=CSV files to open"`
 	Accessor string              `vfilter:"optional,field=accessor,doc=The accessor to use"`
 	Query    vfilter.StoredQuery `vfilter:"required,field=query,doc=query to write into the file."`
 }
@@ -518,7 +518,14 @@ func (self WriteJSONPlugin) Call(
 				return
 			}
 
-			file, err := os.OpenFile(arg.Filename,
+			underlying_file, err := accessors.GetUnderlyingAPIFilename(
+				arg.Accessor, scope, arg.Filename)
+			if err != nil {
+				scope.Log("write_csv: %s", err)
+				return
+			}
+
+			file, err := os.OpenFile(underlying_file,
 				os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0700)
 			if err != nil {
 				scope.Log("write_jsonl: Unable to open file %s: %s",

vql/parsers/leveldb.go

@@ -32,6 +32,7 @@ func (self LevelDBPlugin) Call(
 	scope vfilter.Scope,
 	args *ordereddict.Dict) <-chan vfilter.Row {
 	output_chan := make(chan vfilter.Row)
+
 	go func() {
 		defer close(output_chan)
 		defer utils.RecoverVQL(scope)
@@ -53,54 +54,9 @@ func (self LevelDBPlugin) Call(
 			return
 		}
 
-		var db *leveldb.DB
-		switch arg.Accessor {
-		case "", "auto", "file":
-			db, err = leveldb.OpenFile(arg.Filename.String(), &opt.Options{
-				ReadOnly: true,
-				Strict:   opt.NoStrict,
-			})
-			if err != nil {
-				if !retriableError(err) {
-					scope.Log("leveldb: %v", err)
-					return
-				}
-				scope.Log("DEBUG:leveldb: Directly opening file faild with %v, retrying on a local copy", err)
-				local_path, err1 := maybeMakeLocalCopy(ctx, scope, arg)
-				if err1 != nil {
-					scope.Log("leveldb: %v", err)
-					scope.Log("leveldb: %v", err1)
-					return
-				}
-
-				// Try again with the copy
-				db, err = leveldb.OpenFile(local_path, &opt.Options{
-					ReadOnly: true,
-					Strict:   opt.NoStrict,
-				})
-				if err != nil {
-					scope.Log("leveldb: %v", err)
-					return
-				}
-			}
-
-			// For other accessors we just always make a copy.
-		default:
-			local_path, err := maybeMakeLocalCopy(ctx, scope, arg)
-			if err != nil {
-				scope.Log("leveldb: %v", err)
-				return
-			}
-
-			// Try again with the copy
-			db, err = leveldb.OpenFile(local_path, &opt.Options{
-				ReadOnly: true,
-				Strict:   opt.NoStrict,
-			})
-			if err != nil {
-				scope.Log("leveldb: %v", err)
-				return
-			}
+		db, err := getLevelDBHandle(ctx, scope, arg.Accessor, arg.Filename)
+		if err != nil {
+			return
 		}
 		defer db.Close()
 
@@ -134,25 +90,61 @@ func (self LevelDBPlugin) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *
 	}
 }
 
+func getLevelDBHandle(
+	ctx context.Context, scope vfilter.Scope,
+	accessor string, filename *accessors.OSPath) (
+	db *leveldb.DB, err error) {
+
+	underlying_file, err := accessors.GetUnderlyingAPIFilename(
+		accessor, scope, filename)
+	if err == nil {
+		// Try to open the underlying_file
+		db, err = leveldb.OpenFile(underlying_file, &opt.Options{
+			ReadOnly: true,
+			Strict:   opt.NoStrict,
+		})
+		if err == nil {
+			// Ok it worked, lets use it.
+			return db, nil
+		}
+		scope.Log("DEBUG:leveldb: Directly opening file faild with %v, retrying on a local copy", err)
+	}
+
+	local_path, err := makeLocalCopy(ctx, scope, accessor, filename)
+	if err != nil {
+		scope.Log("leveldb: %v", err)
+		return
+	}
+
+	// Try again with the copy
+	return leveldb.OpenFile(local_path, &opt.Options{
+		ReadOnly: true,
+		Strict:   opt.NoStrict,
+	})
+}
+
 // Maybe make a local copy of the database files.
-func maybeMakeLocalCopy(
+func makeLocalCopy(
 	ctx context.Context, scope vfilter.Scope,
-	arg *LevelDBPluginArgs) (string, error) {
-	accessor, err := accessors.GetAccessor(arg.Accessor, scope)
+	accessor_name string,
+	filename *accessors.OSPath) (string, error) {
+
+	accessor, err := accessors.GetAccessor(accessor_name, scope)
 	if err != nil {
 		return "", err
 	}
 
-	files, err := accessor.ReadDirWithOSPath(arg.Filename)
+	files, err := accessor.ReadDirWithOSPath(filename)
 	if err != nil {
 		return "", err
 	}
 
 	// Create a temp directory to contain all the files.
-	tmpdir_any := (&filesystem.TempdirFunction{}).Call(ctx, scope, ordereddict.NewDict())
+	tmpdir_any := (&filesystem.TempdirFunction{}).Call(
+		ctx, scope, ordereddict.NewDict())
 	tmpdir, ok := tmpdir_any.(string)
 	if !ok {
-		return "", errors.New("Unable to create tempdir")
+		return "", errors.New("leveldb: Unable to create tempdir")
 	}
 
 	total_bytes := 0
@@ -178,7 +170,7 @@ func maybeMakeLocalCopy(
 
 	scope.Log("INFO:leveldb: Copied db %v with accessor %v to local "+
 		"tmp directory %v (Copied %v files, %v bytes)\n",
-		arg.Filename, arg.Accessor, tmpdir, len(files), total_bytes)
+		filename.String(), accessor_name, tmpdir, len(files), total_bytes)
 	return tmpdir, nil
 }