Added profile_memory() and profile_goroutines() VQL functions (#2887)

These present profile results in a more useful structured way so it can
be collected as an artifact better.

Also added a disk usage checker that disables datastore writes when the
disk is too full. This should stop file corruption due to full disk.

actions/query_log.go

@@ -38,6 +38,12 @@ func (self *QueryLogEntry) Close() {
 	self.mu.Lock()
 	defer self.mu.Unlock()
 
+	// Query was already closed - allow Close to be called multiple
+	// times.
+	if self.Duration > 0 {
+		return
+	}
+
 	self.Duration = time.Now().UnixNano() - self.Start.UnixNano()
 
 	// We represent Duration == 0 as not yet complete but sometimes

artifacts/definitions/Elastic/Events/Clients.yaml → artifacts/definitions/Elastic/Events/Upload.yaml

@@ -1,6 +1,7 @@
 name: Elastic.Events.Upload
 aliases:
 - Elastic.Events.Clients
+
 description: |
   This server monitoring artifact will watch a selection of client or
   server monitoring artifacts for new events and push those to an

artifacts/definitions/Generic/Client/Profile.yaml

@@ -27,6 +27,9 @@ description: |
   at the same time since this artifacts itself will not be doing very
   much other than just measuring the state of the process.
 
+  NOTE: As of 0.7.0 release, this artifact will also collect
+  goroutines and heap profiles as distinct sources in a more readable
+  way.
 
 parameters:
   - name: Allocs
@@ -66,6 +69,11 @@ parameters:
     description: Duration of sampling for Profile and Trace.
     default: "30"
 
+export: |
+    LET CleanUp(Name) = regex_replace(
+        re="www.velocidex.com/golang/velociraptor/",
+        replace="", source=Name)
+
 sources:
   - query: |
       SELECT Type,
@@ -77,3 +85,51 @@ sources:
                    logs=Logs, queries=QueryLogs, metrics=Metrics,
                    debug=if(condition=Verbose, then=2, else=1),
                    duration=atoi(string=Duration))
+
+  - name: Goroutines
+    query: |
+      SELECT *, {
+         SELECT format(format="%v (%v:%v)",
+             args=[CleanUp(Name=Name), basename(path=File), Line])
+         FROM CallStack
+         WHERE File =~ 'velociraptor|vfilter|go-ntfs'
+         LIMIT 10
+      } AS CallStack
+      FROM profile_goroutines()
+      WHERE CallStack
+
+  - name: Memory
+    query: |
+      SELECT InUseBytes, InUseObjects, {
+          SELECT format(format="%v (%v:%v)",
+            args=[CleanUp(Name=Name), basename(path=File), Line])
+          FROM CallStack
+          WHERE File =~ 'velociraptor|vfilter|go-ntfs'
+          LIMIT 10
+      } AS CallStack
+      FROM profile_memory()
+      ORDER BY InUseBytes DESC
+
+  - name: Logs
+    query: |
+      SELECT * FROM profile(logs=TRUE)
+
+  - name: RunningQueries
+    query: |
+      SELECT Line.Start AS Timestamp, Line.Query AS Query
+      FROM profile(queries=TRUE)
+      WHERE NOT Line.Duration
+
+  - name: AllQueries
+    query: |
+      SELECT Line.Start AS Timestamp, int(int = Line.Duration / 1000000) AS DurationSec, Line.Query AS Query
+      FROM profile(queries=TRUE)
+
+  - name: Metrics
+    query: |
+      SELECT Line.name AS Name, Line.value as value
+      FROM profile(metrics=TRUE)
+
+column_types:
+  - name: InUseBytes
+    type: mb

artifacts/definitions/Server/Monitor/Health.yaml

@@ -12,10 +12,7 @@ type: SERVER_EVENT
 
 sources:
   - name: Prometheus
-
-    # This artifact is populated by the frontend service using the
-    # total of all frontend metrics.
-    query: SELECT * FROM info() WHERE FALSE
+    query: SELECT sleep(time=10000000) FROM scope()
 
 reports:
   - type: SERVER_EVENT

artifacts/definitions/Server/Monitor/Profile.yaml

@@ -23,6 +23,10 @@ description: |
   kcachegrind profile.grind
   ```
 
+  NOTE: As of 0.7.0 release, this artifact will also collect
+  goroutines and heap profiles as distinct sources in a more readable
+  way.
+
 type: SERVER
 
 parameters:
@@ -63,6 +67,11 @@ parameters:
     description: Duration of sampling for Profile and Trace.
     default: "30"
 
+export: |
+    LET CleanUp(Name) = regex_replace(
+        re="www.velocidex.com/golang/velociraptor/",
+        replace="", source=Name)
+
 sources:
   - query: |
       SELECT Type,
@@ -74,3 +83,52 @@ sources:
                    logs=Logs, queries=QueryLogs, metrics=Metrics,
                    debug=if(condition=Verbose, then=2, else=1),
                    duration=atoi(string=Duration))
+
+  - name: Goroutines
+    query: |
+      SELECT *, {
+         SELECT format(format="%v (%v:%v)",
+             args=[CleanUp(Name=Name), basename(path=File), Line])
+         FROM CallStack
+         WHERE File =~ 'velociraptor|vfilter|go-ntfs'
+         LIMIT 10
+      } AS CallStack
+      FROM profile_goroutines()
+      WHERE CallStack
+
+  - name: Memory
+    query: |
+      SELECT InUseBytes, InUseObjects, {
+          SELECT format(format="%v (%v:%v)",
+            args=[CleanUp(Name=Name), basename(path=File), Line])
+          FROM CallStack
+          WHERE File =~ 'velociraptor|vfilter|go-ntfs'
+          LIMIT 10
+      } AS CallStack
+      FROM profile_memory()
+      ORDER BY InUseBytes DESC
+
+  - name: Logs
+    query: |
+      SELECT * FROM profile(logs=TRUE)
+
+  - name: RunningQueries
+    query: |
+      SELECT Line.Start AS Timestamp, Line.Query AS Query
+      FROM profile(queries=TRUE)
+      WHERE NOT Line.Duration
+
+  - name: AllQueries
+    query: |
+      SELECT Line.Start AS Timestamp, int(int = Line.Duration / 1000000) AS DurationSec, Line.Query AS Query
+      FROM profile(queries=TRUE)
+
+  - name: Metrics
+    query: |
+      SELECT Line.name AS Name, Line.value as value
+      FROM profile(metrics=TRUE)
+
+
+column_types:
+  - name: InUseBytes
+    type: mb