Added FilterRegex and time boxing (#8)

Also added test framework and some tests.

.github/workflows/test.yml

@@ -0,0 +1,22 @@
+name: Test
+on: [pull_request]
+jobs:
+  build:
+    name: Test
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Set up Go 1.19
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Test
+      shell: bash
+      if: always()
+      run: |
+        make
+        go test -v ./...

compile/artifact.go

@@ -88,6 +88,21 @@ parameters:
 - name: CustomGlob
   description: Specify this glob to select other files
 
+- name: DateAfter
+  description: Timebox output to rows after this time.
+  type: timestamp
+  default: "1970-01-01T00:00:00Z"
+
+- name: DateBefore
+  description: Timebox output to rows after this time.
+  type: timestamp
+  default: "2100-01-01T00:00:00Z"
+
+- name: FilterRegex
+  description: Filter critical rows by this regex
+  type: regex
+  default: .
+
 %v
 
 - name: SQLITE_ALWAYS_MAKE_TEMPFILE

compile/compiler.go

@@ -30,6 +30,12 @@ in many types of applications:
 This artifact can hunt for these artifacts in a mostly automated way.
 More info at https://github.com/Velocidex/SQLiteHunter
 
+NOTE: If you want to use this artifact on just a bunch of files already
+collected (for example the files collected using the
+Windows.KapeFiles.Targets artifact) you can use the CustomGlob parameter
+(for example set it to "/tmp/unpacked/**" to consider all files in the
+unpacked directory).
+
 `,
 		Category: ordereddict.NewDict().Set("All", true),
 		Spec: api.Spec{

definitions/Apple_iMessageChat.yaml

@@ -18,6 +18,8 @@ Sources:
   VQL: |
     SELECT timestamp(epoch=date / 1000000000 + 978307200) AS Timestamp, *
     FROM Rows
+    WHERE Timestamp > DateAfter AND Timestamp < DateBefore
+      AND (MessageText, RoomName) =~ FilterRegex
 
   SQL: |
     SELECT

definitions/ChromiumBrowser_AutofillProfiles.yaml

@@ -29,6 +29,9 @@ Sources:
       PhoneNumber, CompanyName, StreetAddress,
       City, State, ZipCode, UseCount, OSPath
     FROM Rows
+    WHERE UseDate > DateAfter AND UseDate < DateBefore
+      AND (FirstName, MiddleName, LastName, EmailAddress, CompanyName, StreetAddress) =~ FilterRegex
+
   SQL: |
     SELECT
       autofill_profiles.guid AS GUID,

definitions/ChromiumBrowser_Cookies.yaml

@@ -28,6 +28,9 @@ Sources:
            Bool(Value=is_persistent) AS IsPersistent,
            Priority, SourcePort, OSPath
     FROM Rows
+    WHERE LastAccessUTC > DateAfter AND LastAccessUTC < DateBefore
+      AND (Name, Path) =~ FilterRegex
+
   SQL: |
     SELECT
       cookies.creation_utc,

definitions/ChromiumBrowser_Extensions.yaml

@@ -38,3 +38,4 @@ Sources:
                 then=upload(file=OSPath.Dirname + GetIcon(Manifest=Manifest))) AS Image,
       Manifest AS _Manifest
     FROM LocaleData
+    WHERE (name, description) =~ FilterRegex

definitions/ChromiumBrowser_Favicons.yaml

@@ -28,7 +28,7 @@ Sources:
          name=format(format="Image%v.png", args=ID)) AS Image,
       OSPath as _OSPath
     FROM Rows
-
+    WHERE LastUpdated > DateAfter AND LastUpdated < DateBefore
 
   SQL: |
     SELECT

definitions/ChromiumBrowser_HistoryVisits.yaml

@@ -29,6 +29,9 @@ Sources:
        visit_duration / 1000000 AS VisitDurationInSeconds,
        OSPath
     FROM Rows
+    WHERE VisitTime > DateAfter
+      AND VisitTime < DateBefore
+      AND (URLTitle, URL) =~ FilterRegex
   SQL: |
     SELECT
       urls.id AS ID,
@@ -72,6 +75,9 @@ Sources:
       get(item=InterruptReason, field=str(str=interrupt_reason), default="Unknown") AS InterruptReason,
       ReferrerURL, SiteURL, TabURL, TabReferrerURL, DownloadURL, OSPath
     FROM Rows
+    WHERE LastAccessTime > DateAfter AND LastAccessTime < DateBefore
+      AND (SiteURL, DownloadURL, TabURL, TabReferrerURL, ReferrerURL, DownloadURL) =~ FilterRegex
+
   SQL: |
     SELECT
       downloads.id AS ID,
@@ -104,6 +110,9 @@ Sources:
        timestamp(winfiletime=(last_visit_time * 10) || 0) AS LastVisitedTime,
        KeywordSearchTerm, Title, URL, OSPath
     FROM Rows
+    WHERE LastVisitedTime > DateAfter AND LastVisitedTime < DateBefore
+      AND (Title, KeywordSearchTerm, URL) =~ FilterRegex
+
   SQL: |
     SELECT
       keyword_search_terms.keyword_id AS KeywordID,

definitions/ChromiumBrowser_Media.yaml

@@ -26,6 +26,9 @@ Sources:
        timestamp(winfiletime=last_updated_time_s || 0) AS LastUpdated,
        OriginID, OSPath
     FROM Rows
+    WHERE LastUpdated > DateAfter AND LastUpdated < DateBefore
+      AND URL =~ FilterRegex
+
   SQL: |
     SELECT
       playback.id AS ID,
@@ -46,6 +49,9 @@ Sources:
       position_ms / 1000 AS PositionInSeconds,
       Title, Artist, Album, SourceTitle, OriginID, OSPath
     FROM Rows
+    WHERE LastUpdated > DateAfter AND LastUpdated < DateBefore
+      AND URL =~ FilterRegex
+
   SQL: |
     SELECT
       playbackSession.id AS ID,

definitions/ChromiumBrowser_NetworkActionPredictor.yaml

@@ -22,6 +22,8 @@ Sources:
 - name: Predictor
   VQL: |
     SELECT * FROM Rows
+    WHERE UserText =~ FilterRegex
+
   SQL: |
     SELECT
        network_action_predictor.id AS ID,

definitions/ChromiumBrowser_OmniboxShortcuts.yaml

@@ -25,6 +25,8 @@ Sources:
       TextTyped, FillIntoEdit, URL, Contents,
       Description, Type, Keyword, TimesSelectedByUser, OSPath
     FROM Rows
+    WHERE LastAccessTime > DateAfter AND LastAccessTime < DateBefore
+      AND (Contents, Description) =~ FilterRegex
 
   SQL: |
     SELECT

definitions/ChromiumBrowser_Sessions.yaml

@@ -17,7 +17,18 @@ Globs:
 Sources:
 - name: Sessions
   VQL: |
-    SELECT * FROM info()
+    SELECT timestamp(winfiletime=(creation_utc * 10) || 0) AS CreationUTC,
+           timestamp(winfiletime=(expires_utc * 10) || 0) AS ExpiresUTC,
+           timestamp(winfiletime=(last_access_utc * 10) || 0) AS LastAccessUTC,
+           HostKey, Name, Path,
+           Bool(Value=is_secure) AS IsSecure,
+           Bool(Value=is_httponly) AS IsHttpOnly,
+           Bool(Value=has_expires) AS HasExpiration,
+           Bool(Value=is_persistent) AS IsPersistent,
+           Priority, SourcePort, OSPath
+    FROM Rows
+    WHERE LastAccessUTC > DateAfter AND LastAccessUTC < DateBefore
+      AND (Name, Path) =~ FilterRegex
 
   SQL: |
     SELECT

definitions/ChromiumBrowser_TopSites.yaml

@@ -21,6 +21,7 @@ Globs:
 Sources:
 - VQL: |
     SELECT * FROM Rows
+    WHERE ( URL =~ FilterRegex OR Title =~ FilterRegex )
 
   SQL: |
     SELECT

definitions/Firefox_Bookmarks.yaml

@@ -26,6 +26,9 @@ Sources:
        timestamp(epoch=lastModified) AS LastModified,
        Position, Title, URL, ForeignKey, OSPath
     FROM Rows
+    WHERE LastModified > DateAfter AND LastModified < DateBefore
+      AND (Title, URL) =~ FilterRegex
+
   SQL: |
     SELECT
       Bookmarks.id AS ID,
@@ -48,6 +51,9 @@ Sources:
        timestamp(epoch=lastModified) AS LastModified,
        OSPath
     FROM Rows
+    WHERE LastModified > DateAfter AND LastModified < DateBefore
+      AND Content =~ FilterRegex
+
   SQL: |
     SELECT
       moz_annos.place_id AS PlaceID,
@@ -69,9 +75,12 @@ Sources:
        VisitCount, URL, Title, Description,
        get(item= VisitType, field=str(str=visit_type), default="Unknown") AS VisitType,
        Bool(Value=hidden) AS Hidden,
-       Bool(Value=types) AS Typed,
+       Bool(Value=typed) AS Typed,
        Frecency, PreviewImageURL, OSPath
     FROM Rows
+    WHERE LastVisitDate > DateAfter AND LastVisitDate < DateBefore
+      AND (Title, URL, Description) =~ FilterRegex
+
   SQL: |
     SELECT
       moz_historyvisits.id AS VisitID,
@@ -83,7 +92,7 @@ Sources:
       moz_places.description AS Description,
       moz_historyvisits.visit_type,
       moz_places.hidden,
-      moz_places.types,
+      moz_places.typed,
       moz_places.frecency AS Frecency,
       moz_places.preview_image_url AS PreviewImageURL
     FROM moz_places

definitions/Firefox_Cookies.yaml

@@ -25,6 +25,10 @@ Sources:
        Bool(Value= isSecure) AS IsSecure,
        Bool(Value= isHttpOnly) AS IsHTTPOnly, OSPath
     FROM Rows
+    WHERE LastAccessedTime > DateAfter
+      AND LastAccessedTime < DateBefore
+      AND ( Name =~ FilterRegex OR Value =~ FilterRegex )
+
   SQL: |
     SELECT
       moz_cookies.id AS ID,

definitions/Firefox_Downloads.yaml

@@ -24,6 +24,10 @@ Sources:
        timestamp(epoch= expiry) AS Expiration,
        CurrentBytes, MaxBytes, OSPath
     FROM Rows
+    WHERE StartTime > DateAfter
+      AND StartTime < DateBefore
+      AND Name =~ FilterRegex
+
   SQL: |
     SELECT
        moz_downloads.id AS ID,

definitions/Firefox_FormHistory.yaml

@@ -24,6 +24,9 @@ Sources:
        timestamp(epoch= lastUsed) AS LastUsed,
        GUID, OSPath
     FROM Rows
+    WHERE LastUsed > DateAfter AND LastUsed < DateBefore
+      AND ( FieldName =~ FilterRegex OR Value =~ FilterRegex )
+
   SQL: |
     SELECT
       id AS ID,

definitions/InternetExplorer_WebCacheV01.yaml

@@ -30,10 +30,15 @@ Sources:
     SELECT * FROM foreach(row=MatchingFiles, query={
       SELECT * FROM AllHits(OSPath=OSPath)
     })
+    WHERE AccessedTime > DateAfter AND AccessedTime < DateBefore
+      AND Url =~ FilterRegex
+
 
 - name: Highlights
   VQL: |
     SELECT * FROM foreach(row=MatchingFiles, query={
       SELECT AccessedTime, ModifiedTime, ExpiryTime, Url
       FROM AllHits(OSPath=OSPath)
     })
+    WHERE AccessedTime > DateAfter AND AccessedTime < DateBefore
+      AND Url =~ FilterRegex

definitions/MacOS_Applications_Cache.yaml

@@ -37,6 +37,8 @@ Sources:
        partition AS Partition,
        OSPath
     FROM Rows
+    WHERE Timestamp > DateAfter AND Timestamp < DateBefore
+      AND Application =~ FilterRegex
 
   SQL: |
     SELECT cfurl_cache_response.entry_ID AS entry_ID,

definitions/MacOS_Notes.yaml

@@ -56,6 +56,8 @@ Sources:
      gunzip(string=Data) AS Data,
      OSPath
     FROM Rows
+    WHERE LastOpenedTime > DateAfter AND LastOpenedTime < DateBefore
+      AND ( Title =~ FilterRegex OR Data =~ FilterRegex )
 
   SQL: |
     SELECT n.Z_PK AS Key,

definitions/Windows_ActivitiesCache.yaml

@@ -23,6 +23,7 @@ Sources:
       ActivityId[8:10], ActivityId[10:] ]) AS ActivityId,
       Platform, PackageName, ExpirationTime, OSPath
     FROM Rows
+
   SQL: |
     Select ActivityId, Platform, PackageName, ExpirationTime
     FROM Activity_PackageId
@@ -33,8 +34,8 @@ Sources:
   VQL: |
       SELECT
         CreatedTime,
-        LastModifiedTime,
-        LastModifiedOnClient,
+        timestamp(epoch=LastModifiedTime) AS LastModifiedTime,
+        timestamp(epoch=LastModifiedOnClient) AS LastModifiedOnClient,
         StartTime,
         EndTime,
         Payload,
@@ -43,3 +44,6 @@ Sources:
         OSPath AS Path,
         Mtime
       FROM Rows
+      WHERE StartTime > DateAfter
+        AND StartTime < DateBefore
+        AND ClipboardPayload =~ FilterRegex

definitions/Windows_SearchService.yaml

@@ -31,13 +31,16 @@ Sources:
           FileName
        FROM parse_ese(file=OSPath, table= "SystemIndex_Gthr")
     })
+    WHERE LastModified > DateAfter AND LastModified < DateBefore
+      AND FileName =~ FilterRegex
 
 - name: SystemIndex_GthrPth
   VQL: |
     SELECT * FROM foreach(row=MatchingFiles, query={
        SELECT Scope, Parent, Name
        FROM parse_ese(file=OSPath, table= "SystemIndex_GthrPth")
     })
+    WHERE Name =~ FilterRegex
 
 - name: SystemIndex_PropertyStore
   VQL: |
@@ -68,6 +71,7 @@ Sources:
        SELECT *
        FROM PropStore(OSPath=OSPath)
     })
+    WHERE System_DateAccessed > DateAfter AND System_DateAccessed < DateBefore
 
 - name: SystemIndex_PropertyStore_Highlights
   VQL: |
@@ -84,6 +88,7 @@ Sources:
           X.System_Search_AutoSummary AS System_Search_AutoSummary
        FROM PropStore(OSPath=OSPath)
     })
+    WHERE System_DateAccessed > DateAfter AND System_DateAccessed < DateBefore
 
 - name: BrowsingActivity
   VQL: |
@@ -105,3 +110,5 @@ Sources:
        FROM PropStore(OSPath=OSPath)
        WHERE ActivityHistory_AppId
     })
+    WHERE ActivityHistory_StartTime > DateAfter
+      AND ActivityHistory_StartTime < DateBefore

go.mod

@@ -2,11 +2,23 @@ module github.com/Velocidex/SQLiteHunter
 
 go 1.20
 
+require (
+	github.com/Velocidex/ordereddict v0.0.0-20221110130714-6a7cb85851cd
+	github.com/alecthomas/assert v1.0.0
+	github.com/sebdah/goldie/v2 v2.5.3
+	github.com/stretchr/testify v1.8.4
+	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
+)
+
 require (
 	github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a // indirect
-	github.com/Velocidex/ordereddict v0.0.0-20221110130714-6a7cb85851cd // indirect
 	github.com/Velocidex/yaml/v2 v2.2.8 // indirect
+	github.com/alecthomas/colour v0.1.0 // indirect
+	github.com/alecthomas/repr v0.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 )