Merge pull request #95 from BenB196/staging

Staging to Master

VERSION

@@ -1 +1 @@
-0.2.4
+0.2.5

eventOutput/fileHandler.go

@@ -82,6 +82,9 @@ type SemiElasticFileEvent struct {
 	IdentifiedExtensionMIMEType string     `json:"identified_extension_mime_type,omitempty"`
 	CurrentExtensionMIMEType    string     `json:"current_extension_mime_type,omitempty"`
 	SuspiciousFileTypeMismatch  *bool      `json:"suspicious_file_type_mismatch,omitempty"`
+	PrintJobName                string     `json:"print_job_name,omitempty"`
+	PrinterName                 string     `json:"printer_name,omitempty"`
+	PrintedFilesBackupPath      string     `json:"printed_files_backup_path,omitempty"`
 }
 
 type ElasticFileEvent struct {
@@ -95,6 +98,7 @@ type ElasticFileEvent struct {
 	Tab            *Tab            `json:"tab,omitempty"`
 	RemovableMedia *RemovableMedia `json:"removable_media,omitempty"`
 	EmailDlp       *EmailDlp       `json:"email_dlp,omitempty"`
+	Printing       *Printing       `json:"printing,omitempty"`
 }
 
 type Event struct {
@@ -113,7 +117,18 @@ type Hash struct {
 }
 
 type URL struct {
-	Full string `json:"full,omitempty"`
+	Full             string `json:"full,omitempty"`
+	Domain           string `json:"domain,omitempty"`
+	Extension        string `json:"extension,omitempty"`
+	Fragment         string `json:"fragment,omitempty"`
+	Path             string `json:"path,omitempty"`
+	Port             *int   `json:"port,omitempty"`
+	Query            string `json:"query,omitempty"`
+	Scheme           string `json:"scheme,omitempty"`
+	Username         string `json:"username,omitempty"`
+	Password         string `json:"password,omitempty"`
+	RegisteredDomain string `json:"registered_domain,omitempty"`
+	TopLevelDomain   string `json:"top_level_domain,omitempty"`
 }
 
 type File struct {
@@ -203,6 +218,16 @@ type EmailDlp struct {
 	Recipients  []string `json:"recipients,omitempty"`
 }
 
+type Printing struct {
+	JobName                string   `json:"job_name,omitempty"`
+	Printer                *Printer `json:"printer,omitempty"`
+	PrintedFilesBackupPath string   `json:"printed_files_backup_path,omitempty"`
+}
+
+type Printer struct {
+	Name string `json:"name,omitempty"`
+}
+
 type Geo struct {
 	Status        string    `json:"status,omitempty"`
 	Message       string    `json:"message,omitempty"`

ffsEvent/ffsFetcher.go

@@ -118,7 +118,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg
 				//convert to valid IP addresses if enabled
 				if query.ValidIpAddressesOnly && len(ffsEvent.PrivateIpAddresses) > 0 {
 					for x, privateIpAddress := range ffsEvent.PrivateIpAddresses {
-						ffsEvent.PrivateIpAddresses[x] = strings.Split(privateIpAddress,"%")[0]
+						ffsEvent.PrivateIpAddresses[x] = strings.Split(privateIpAddress, "%")[0]
 					}
 				}
 
@@ -154,11 +154,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg
 						hash = nil
 					}
 
-					url := &eventOutput.URL{Full: ffsEvent.Url}
-
-					if *url == (eventOutput.URL{}) {
-						url = nil
-					}
+					url := getUrlInfo(ffsEvent.Url)
 
 					file := &eventOutput.File{
 						Path:                        ffsEvent.FilePath,
@@ -293,11 +289,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg
 						process = nil
 					}
 
-					tabUrl := &eventOutput.URL{Full: ffsEvent.TabUrl}
-
-					if *tabUrl == (eventOutput.URL{}) {
-						tabUrl = nil
-					}
+					tabUrl := getUrlInfo(ffsEvent.TabUrl)
 
 					tab := &eventOutput.Tab{
 						WindowTitle: ffsEvent.TabWindowTitle,
@@ -335,6 +327,24 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg
 						emailDlp = nil
 					}
 
+					printer := &eventOutput.Printer{
+						Name: ffsEvent.PrinterName,
+					}
+
+					if ffsEvent.PrinterName == "" {
+						printer = nil
+					}
+
+					printing := &eventOutput.Printing{
+						JobName:                ffsEvent.PrintJobName,
+						Printer:                printer,
+						PrintedFilesBackupPath: ffsEvent.PrintedFilesBackupPath,
+					}
+
+					if *printing == (eventOutput.Printing{}) {
+						printing = nil
+					}
+
 					elasticFileEvent := &eventOutput.ElasticFileEvent{
 						Event:          event,
 						Timestamp:      timestamp,
@@ -346,6 +356,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg
 						Tab:            tab,
 						RemovableMedia: removableMedia,
 						EmailDlp:       emailDlp,
+						Printing:       printing,
 					}
 
 					elasticFFSEvents = append(elasticFFSEvents, *elasticFileEvent)
@@ -408,6 +419,9 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg
 						IdentifiedExtensionMIMEType: ffsEvent.IdentifiedExtensionMIMEType,
 						CurrentExtensionMIMEType:    ffsEvent.CurrentExtensionMIMEType,
 						SuspiciousFileTypeMismatch:  ffsEvent.SuspiciousFileTypeMismatch,
+						PrintJobName:                ffsEvent.PrintJobName,
+						PrinterName:                 ffsEvent.PrinterName,
+						PrintedFilesBackupPath:      ffsEvent.PrintedFilesBackupPath,
 					}
 
 					var semiElasticFFSEvent eventOutput.SemiElasticFFSEvent
@@ -898,7 +912,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg
 		panic(err)
 	}
 
-	promMetrics.IncrementEventsProcessed(len(ffsEvents))
+	promMetrics.IncrementEventsProcessed(len(*fileEvents))
 	promMetrics.DecreaseInProgressQueries()
 	endTime := time.Now()
 	duration := endTime.Sub(startTime)
@@ -918,4 +932,4 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg
 		retryGetFileEventsDuration.String() + " - Enrichment Duration: " + enrichmentDuration.String() + " - Output Duration: " + outputDuration.String() +
 		" - Write Last Completed Query Duration: " + writeLastCompletedQueryDuration.String() +
 		" - Remove In Progress Query Duration: " + removeInProgressQueryDuration.String() + " - Duration: " + duration.String())
-}
+}

ffsEvent/ffsQuery.go

@@ -129,7 +129,7 @@ func FFSQuery(configuration config.Config, query config.FFSQuery) {
 		for {
 			select {
 			case <-queryIntervalTimeTicker.C:
-				if *query.MaxConcurrentQueries == -1 || len(inProgressQueries) < *query.MaxConcurrentQueries {
+				if *query.MaxConcurrentQueries == -1 || len(inProgressQueries) <= *query.MaxConcurrentQueries {
 					go queryFetcher(query, &inProgressQueries, *authData, configuration, &lastCompletedQuery, maxTime, false, elasticClient, ctx, quit, 0, false)
 				} else {
 					log.Println("Rate limiting query: " + query.Name)

ffsEvent/ffsUtils.go

@@ -5,6 +5,12 @@ import (
 	"github.com/BenB196/crashplan-ffs-go-pkg"
 	"github.com/BenB196/crashplan-ffs-puller/config"
 	"github.com/BenB196/crashplan-ffs-puller/eventOutput"
+	"golang.org/x/net/publicsuffix"
+	"log"
+	"net"
+	"net/url"
+	"strconv"
+	"strings"
 	"time"
 )
 
@@ -161,4 +167,71 @@ func getNewerTimeQuery(lastInProgressQuery eventOutput.InProgressQuery, lastComp
 	} else {
 		return lastCompletedQuery
 	}
+}
+
+func getUrlInfo(urlFull string) *eventOutput.URL {
+	if urlFull == "" {
+		return nil
+	}
+
+	var eventUrl eventOutput.URL
+
+	eventUrl.Full = urlFull
+
+	u, err := url.Parse(urlFull)
+
+	if err != nil {
+		log.Println("Error processing Event URL; error: " + err.Error() + ", URL: " + urlFull)
+	} else {
+		//set scheme
+		eventUrl.Scheme = u.Scheme
+		//set username
+		eventUrl.Username = u.User.Username()
+		//set password
+		eventUrl.Password, _ = u.User.Password()
+		//get host/port if : in string
+		if strings.Contains(u.Host,":") {
+			host, port, err := net.SplitHostPort(u.Host)
+			if err != nil {
+				log.Println("Error processing host from Event Url; error: " + err.Error() + ", Host: " + u.Host)
+			}
+			//set domain
+			eventUrl.Domain = host
+			//set port
+			if port == "" {
+				eventUrl.Port = nil
+			} else {
+				portNum, err := strconv.Atoi(port)
+				if err != nil {
+					log.Println("Error processing port from Event Url; error: " + err.Error() + ", Port: " + port)
+				}
+				eventUrl.Port = &portNum
+			}
+		} else {
+			//set domain
+			eventUrl.Domain = u.Host
+			//set port
+			eventUrl.Port = nil
+		}
+
+		//set path
+		eventUrl.Path = u.Path
+		//set extension
+		if strings.Contains(eventUrl.Path, ".") {
+			pathParts := strings.Split(eventUrl.Path,".")
+			eventUrl.Extension = pathParts[len(pathParts)-1]
+		}
+		//set fragment
+		eventUrl.Fragment = u.Fragment
+		//set query
+		eventUrl.Query = u.RawQuery
+		//set registered domain
+		eventUrl.RegisteredDomain, err = publicsuffix.EffectiveTLDPlusOne(eventUrl.Domain)
+		if err != nil {
+			log.Println("Error getting Registered Domain; error: " + err.Error() + ", Domain: " + eventUrl.Domain)
+		}
+		//set top level domain
+		eventUrl.TopLevelDomain, _ = publicsuffix.PublicSuffix(eventUrl.Domain)
+	}
+	return &eventUrl
 }

go.mod

@@ -3,10 +3,10 @@ module github.com/BenB196/crashplan-ffs-puller
 go 1.14
 
 require (
-	github.com/BenB196/crashplan-ffs-go-pkg v0.1.6
+	github.com/BenB196/crashplan-ffs-go-pkg v0.1.7
 	github.com/BenB196/ip-api-go-pkg v0.0.8
 	github.com/VictoriaMetrics/fastcache v1.5.7
-	github.com/google/go-cmp v0.4.0
 	github.com/olivere/elastic/v7 v7.0.14
 	github.com/prometheus/client_golang v1.5.1
+	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
 )

go.sum

@@ -32,6 +32,8 @@ github.com/BenB196/crashplan-ffs-go-pkg v0.1.5 h1:CshGuCzcGpY5nmn8zKkbJ8wDvP3nNA
 github.com/BenB196/crashplan-ffs-go-pkg v0.1.5/go.mod h1:8ikK7PskHvmZupmN1uUnL3+z/CeIsa+7ibvfPuY34Dg=
 github.com/BenB196/crashplan-ffs-go-pkg v0.1.6 h1:q3T0ElfvId+TM2Qvf8p+L86GA08U2y0SCI79ZbO0mR8=
 github.com/BenB196/crashplan-ffs-go-pkg v0.1.6/go.mod h1:8ikK7PskHvmZupmN1uUnL3+z/CeIsa+7ibvfPuY34Dg=
+github.com/BenB196/crashplan-ffs-go-pkg v0.1.7 h1:rn60QBcqLswveJX/xP2da+E2T3XBh6v0LDH+Co7aTsE=
+github.com/BenB196/crashplan-ffs-go-pkg v0.1.7/go.mod h1:8ikK7PskHvmZupmN1uUnL3+z/CeIsa+7ibvfPuY34Dg=
 github.com/BenB196/ip-api-go-pkg v0.0.3 h1:FWmM7FkhT1N55jd4jPW7W9LVOQrG89DLlrgwMb5fosw=
 github.com/BenB196/ip-api-go-pkg v0.0.3/go.mod h1:ccPdkBNnzf/uvuk7qXgEO06TCS/qILNJQP/KETQG4jU=
 github.com/BenB196/ip-api-go-pkg v0.0.4 h1:BbWELxooG6l2gaXQ4i4gm6NsyikdbhalF+TKg08gDPQ=
@@ -218,6 +220,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=