Sanitize artifact and sources names

.github/workflows/test.yml

@@ -19,8 +19,8 @@ jobs:
       id: velociraptor
       with:
         repository: velocidex/velociraptor
-        tag: v0.75
-        fileName: "velociraptor-v0.75.2-linux-amd64-musl"
+        tag: v0.77.3
+        fileName: "velociraptor-v0.77.3-linux-amd64-musl.gz"
         out-file-path: tests
 
     - name: Build artifacts
@@ -29,7 +29,8 @@ jobs:
 
     - name: Run Velociraptor
       run: |
-        mv ${{ fromJson(steps.velociraptor.outputs.downloaded_files)[0]}} ./tests/velociraptor.bin
+        mv ${{ fromJson(steps.velociraptor.outputs.downloaded_files)[0]}} ./tests/velociraptor.bin.gz
+        gunzip ./tests/velociraptor.bin.gz
         chmod +x ./tests/velociraptor.bin
         ls -l ./output/
         ./tests/velociraptor.bin artifacts verify --builtin -v ./output/*.yaml

compile/artifact.go

@@ -11,6 +11,7 @@ import (
 	"text/template"
 
 	"github.com/Velocidex/SQLiteHunter/api"
+	"github.com/Velocidex/SQLiteHunter/definitions"
 	"github.com/Velocidex/ordereddict"
 
 	_ "embed"
@@ -42,6 +43,7 @@ func (self *Artifact) Yaml() (string, error) {
 			"Quote": func(in interface{}) interface{} {
 				return fmt.Sprintf("%q", in)
 			},
+			"Sanitize": definitions.SanitizeName,
 			"DictRange": func(in interface{}) interface{} {
 				var res []dictValue
 

compile/template.yaml

@@ -196,7 +196,7 @@ sources:
   {{ $action = "ApplyFile" }}
 {{ end -}}
 
-- name: {{ Quote $v.Key }}
+- name: {{ Sanitize $v.Key }}
   notebook:
     - type: none
       output: "{{ $v.Key }} - Recalculate to view results"

definitions/Apple_IntelligencePlatform_Wifi.yaml

@@ -21,6 +21,8 @@ SQLiteIdentifyValue: 1
 
 Sources:
   - name: Wifi
+    Preamble: |
+      // linter: symbol_mask_warn:timestamp
     SQL: |
       SELECT behaviorType, behaviorIdentifier, timestamp
       FROM wifiContextEvents ORDER BY timestamp

definitions/MacOS_Applications_Cache.yaml

@@ -23,7 +23,10 @@ Globs:
   - "/Users/*/Library/Caches/*/Cache.db"
 
 Sources:
-- VQL: |
+- Preamble: |
+    // linter: symbol_mask_warn:version
+
+  VQL: |
     SELECT
        time_stamp AS Timestamp,
        OSPath.Base AS Application,

definitions/load.go

@@ -2,16 +2,24 @@ package definitions
 
 import (
 	"fmt"
+	"io"
 	"io/fs"
-	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/Velocidex/SQLiteHunter/api"
 	"gopkg.in/yaml.v3"
 )
 
+// Artifact names are more strict now so we need to clean them up.
+var artifactNameRegex = regexp.MustCompile("[^a-zA-Z0-9_.]")
+
+func SanitizeName(name string) string {
+	return artifactNameRegex.ReplaceAllLiteralString(name, "_")
+}
+
 func LoadDefinitions(root string) ([]api.Definition, error) {
 	var definitions []api.Definition
 
@@ -25,7 +33,7 @@ func LoadDefinitions(root string) ([]api.Definition, error) {
 					return nil
 				}
 
-				data, err := ioutil.ReadAll(fd)
+				data, err := io.ReadAll(fd)
 				if err != nil {
 					fmt.Printf("Error processing %v: %v\n", path, err)
 					return nil
@@ -39,10 +47,19 @@ func LoadDefinitions(root string) ([]api.Definition, error) {
 				definition.Filename_ = path
 				definition.RawData_ = string(data)
 
-				definitions = append(definitions, definition)
+				definitions = append(definitions, sanitizeDefinition(definition))
 			}
 			return nil
 		})
 
 	return definitions, err
 }
+
+func sanitizeDefinition(definition api.Definition) api.Definition {
+	definition.Name = SanitizeName(definition.Name)
+	for _, s := range definition.Sources {
+		s.Name = SanitizeName(s.Name)
+	}
+
+	return definition
+}

testing/fixtures/TestArtifact.golden

@@ -1,59 +1,59 @@
 {
   "All Records": [
-    "[][",
+    "[",
     " {",
     "  \"LastVisitDate\": \"2020-06-27T09:29:54.51375Z\",",
     "  \"URL\": \"https://www.mozilla.org/privacy/firefox/\",",
     "  \"Description\": null,",
-    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
+    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox_Places_History\"",
     " },",
     " {",
     "  \"LastVisitDate\": \"2020-06-27T09:30:05.721357Z\",",
     "  \"URL\": \"http://github.com/seanbreckenridge/dotfiles\",",
     "  \"Description\": null,",
-    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
+    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox_Places_History\"",
     " },",
     " {",
     "  \"LastVisitDate\": \"2020-06-30T05:53:37.171Z\",",
     "  \"URL\": \"https://www.mozilla.org/en-US/firefox/78.0a2/firstrun/\",",
     "  \"Description\": \"Firefox Developer Edition is the blazing fast browser that offers cutting edge developer tools and latest features like CSS Grid support and framework debugging\",",
-    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
+    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox_Places_History\"",
     " },",
     " {",
     "  \"LastVisitDate\": \"2021-02-21T08:55:10.488Z\",",
     "  \"URL\": \"https://www.mozilla.org/en-US/privacy/firefox/\",",
     "  \"Description\": \"\\n  Our Privacy Notices describe the data our products and services receive, share, and use, as well as choices available to you.\\n\",",
-    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
+    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox_Places_History\"",
     " }",
     "]"
   ],
   "After 2021-02-20T08:55:10.488Z should be only 2021-02-21T08:55:10.488Z": [
-    "[][",
+    "[",
     " {",
     "  \"LastVisitDate\": \"2021-02-21T08:55:10.488Z\",",
     "  \"URL\": \"https://www.mozilla.org/en-US/privacy/firefox/\",",
     "  \"Description\": \"\\n  Our Privacy Notices describe the data our products and services receive, share, and use, as well as choices available to you.\\n\",",
-    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
+    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox_Places_History\"",
     " }",
     "]"
   ],
   "DateBefore=2020-06-27T09:30:00Z should be only 2020-06-27T09:29:54.51375Z": [
-    "[][",
+    "[",
     " {",
     "  \"LastVisitDate\": \"2020-06-27T09:29:54.51375Z\",",
     "  \"URL\": \"https://www.mozilla.org/privacy/firefox/\",",
     "  \"Description\": null,",
-    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
+    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox_Places_History\"",
     " }",
     "]"
   ],
   "FilterRegex=Firefox Developer Edition": [
-    "[][",
+    "[",
     " {",
     "  \"LastVisitDate\": \"2020-06-30T05:53:37.171Z\",",
     "  \"URL\": \"https://www.mozilla.org/en-US/firefox/78.0a2/firstrun/\",",
     "  \"Description\": \"Firefox Developer Edition is the blazing fast browser that offers cutting edge developer tools and latest features like CSS Grid support and framework debugging\",",
-    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox Places History\"",
+    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Firefox_Places_History\"",
     " }",
     "]"
   ]

testing/fixtures/TestEdgeWebAssistDatabase.golden

@@ -1,6 +1,6 @@
 {
   "FilterRegex=Audio": [
-    "[][",
+    "[",
     " {",
     "  \"_Source\": \"Generic.Forensic.SQLiteHunter/AllFiles\",",
     " },",
@@ -10,7 +10,7 @@
     "  \"Title\": \"How to Record Audio on Windows 10\",",
     "  \"URL\": \"https://www.lifewire.com/how-to-record-audio-on-windows-10-4773840\",",
     "  \"VisitCount\": 1,",
-    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Edge Browser Navigation History_Navigation History\",",
+    "  \"_Source\": \"Generic.Forensic.SQLiteHunter/Edge_Browser_Navigation_History_Navigation_History\",",
     " }",
     "]"
   ]

testing/sqlitehunter_test.go

@@ -1,6 +1,7 @@
 package testing
 
 import (
+	"compress/gzip"
 	"fmt"
 	"io"
 	"net/http"
@@ -18,7 +19,7 @@ import (
 )
 
 const (
-	VelociraptorUrl        = "https://github.com/Velocidex/velociraptor/releases/download/v0.74/velociraptor-v0.74.5-linux-amd64-musl"
+	VelociraptorUrl        = "https://github.com/Velocidex/velociraptor/releases/download/v0.77.3/velociraptor-v0.77.3-linux-amd64-musl.gz"
 	VelociraptorBinaryPath = "./velociraptor.bin"
 )
 
@@ -48,7 +49,10 @@ func (self *SQLiteHunterTestSuite) findAndPrepareBinary() {
 		defer out.Close()
 
 		// Write the body to file
-		_, err = io.Copy(out, resp.Body)
+		gz, err := gzip.NewReader(resp.Body)
+		assert.NoError(t, err)
+
+		_, err = io.Copy(out, gz)
 		assert.NoError(t, err)
 	}
 }

testing/testcases/chrome_notifications.out.yaml

@@ -12,14 +12,14 @@ Output: [
   "Site": "https://docs.velociraptor.app:443,*",
   "LastModified": "2024-01-22T04:19:19Z",
   "LastEngagementTime": "2024-01-21T20:10:29Z",
-  "_Source": "Generic.Forensic.SQLiteHunter/Chromium Browser Notifications_Site Engagements",
+  "_Source": "Generic.Forensic.SQLiteHunter/Chromium_Browser_Notifications_Site_Engagements",
   "OSPath": "Preferences"
  },
  {
   "Site": "https://github.com:443,*",
   "LastModified": "2024-01-22T04:19:19Z",
   "LastEngagementTime": "2024-01-21T20:17:51Z",
-  "_Source": "Generic.Forensic.SQLiteHunter/Chromium Browser Notifications_Site Engagements",
+  "_Source": "Generic.Forensic.SQLiteHunter/Chromium_Browser_Notifications_Site_Engagements",
   "OSPath": "Preferences"
  },
  {
@@ -30,7 +30,7 @@ Output: [
    "CouldShowBannerEvents": "2024-01-21T14:33:10Z",
    "LastShown": "2024-01-21T14:32:36Z"
   },
-  "_Source": "Generic.Forensic.SQLiteHunter/Chromium Browser Notifications_App Banners",
+  "_Source": "Generic.Forensic.SQLiteHunter/Chromium_Browser_Notifications_App_Banners",
   "OSPath": "Preferences"
  },
  {
@@ -41,7 +41,7 @@ Output: [
    "CouldShowBannerEvents": "2024-01-23T01:36:02Z",
    "LastShown": "2024-01-23T01:35:54Z"
   },
-  "_Source": "Generic.Forensic.SQLiteHunter/Chromium Browser Notifications_App Banners",
+  "_Source": "Generic.Forensic.SQLiteHunter/Chromium_Browser_Notifications_App_Banners",
   "OSPath": "Preferences"
  },
  {
@@ -52,7 +52,7 @@ Output: [
    "CouldShowBannerEvents": "2024-01-21T14:48:57Z",
    "LastShown": "2024-01-21T14:48:23Z"
   },
-  "_Source": "Generic.Forensic.SQLiteHunter/Chromium Browser Notifications_App Banners",
+  "_Source": "Generic.Forensic.SQLiteHunter/Chromium_Browser_Notifications_App_Banners",
   "OSPath": "Preferences"
  },
  {
@@ -63,14 +63,14 @@ Output: [
    "CouldShowBannerEvents": "2024-01-21T14:37:03Z",
    "LastShown": "2024-01-21T14:36:56Z"
   },
-  "_Source": "Generic.Forensic.SQLiteHunter/Chromium Browser Notifications_App Banners",
+  "_Source": "Generic.Forensic.SQLiteHunter/Chromium_Browser_Notifications_App_Banners",
   "OSPath": "Preferences"
  },
  {
   "Site": "https://appuals.com:443,*",
   "LastModified": "2019-10-14T06:39:30Z",
   "Setting": "Block",
-  "_Source": "Generic.Forensic.SQLiteHunter/Chromium Browser Notifications_Notification Preferences",
+  "_Source": "Generic.Forensic.SQLiteHunter/Chromium_Browser_Notifications_Notification_Preferences",
   "OSPath": "Preferences"
  }
 ]

testing/testcases/edge_webassist.out.yaml

@@ -14,7 +14,7 @@ Output: [
   "Title": "How to Record Audio on Windows 10",
   "URL": "https://www.lifewire.com/how-to-record-audio-on-windows-10-4773840",
   "VisitCount": 1,
-  "_Source": "Generic.Forensic.SQLiteHunter/Edge Browser Navigation History_Navigation History",
+  "_Source": "Generic.Forensic.SQLiteHunter/Edge_Browser_Navigation_History_Navigation_History",
   "OSPath": "WebAssistDatabase"
  }
 ]