From dcd4cf43f81d5540bd29f3641cb4dbe6411b8bb5 Mon Sep 17 00:00:00 2001 From: BenB196 <8277432+BenB196@users.noreply.github.com> Date: Tue, 6 Oct 2020 11:30:34 -0400 Subject: [PATCH 1/5] Removed support for Elasticsearch Half Output --- eventOutput/fileHandler.go | 71 ------------- ffsEvent/ffsFetcher.go | 197 ------------------------------------- 2 files changed, 268 deletions(-) diff --git a/eventOutput/fileHandler.go b/eventOutput/fileHandler.go index cec04ab..27d5391 100644 --- a/eventOutput/fileHandler.go +++ b/eventOutput/fileHandler.go @@ -20,77 +20,6 @@ type FFSEvent struct { GeoLocation *Location `json:"geoPoint,omitempty"` } -type SemiElasticFFSEvent struct { - FileEvent SemiElasticFileEvent `json:"file_event"` - Geo *Geo `json:"geo,omitempty"` -} - -type SemiElasticFileEvent struct { - EventId string `json:"event_id"` - EventType string `json:"event_type"` - EventTimestamp *time.Time `json:"event_timestamp,omitempty"` - InsertionTimestamp *time.Time `json:"insertion_timestamp,omitempty"` - FilePath string `json:"file_path,omitempty"` - FileName string `json:"file_name"` - FileType string `json:"file_type,omitempty"` - FileCategory string `json:"file_category,omitempty"` - IdentifiedExtensionCategory string `json:"identified_extension_category,omitempty"` - CurrentExtensionCategory string `json:"current_extension_category,omitempty"` - FileSize *int `json:"file_size"` - FileOwner []string `json:"file_owner,omitempty"` //Array of owners - Md5Checksum string `json:"md5_checksum,omitempty"` - Sha256Checksum string `json:"sha256_checksum,omitempty"` - CreatedTimestamp *time.Time `json:"created_timestamp,omitempty"` - ModifyTimestamp *time.Time `json:"modify_timestamp,omitempty"` - DeviceUsername string `json:"device_username,omitempty"` - DeviceUid string `json:"device_uid,omitempty"` - UserUid string `json:"user_uid,omitempty"` - OsHostname string `json:"os_hostname,omitempty"` - DomainName string `json:"domain_name,omitempty"` - PublicIpAddress string `json:"public_ip_address,omitempty"` - PrivateIpAddresses []string `json:"private_ip_addresses,omitempty"` //Array of IP address strings - Actor string `json:"actor,omitempty"` - DirectoryId []string `json:"directory_id,omitempty"` //An array of something, I am not sure - Source string `json:"source,omitempty"` - Url string `json:"url,omitempty"` - Shared *bool `json:"shared,omitempty"` - SharedWith []string `json:"shared_with,omitempty"` //An array of strings (Mainly Email Addresses) - SharingTypeAdded []string `json:"sharing_type_added,omitempty"` - CloudDriveId string `json:"cloud_drive_id,omitempty"` - DetectionSourceAlias string `json:"detection_source_alias,omitempty"` - FileId string `json:"file_id,omitempty"` - Exposure []string `json:"exposure,omitempty"` - ProcessOwner string `json:"process_owner,omitempty"` - ProcessName string `json:"process_name,omitempty"` - TabWindowTitle string `json:"tab_window_title,omitempty"` - TabUrl string `json:"tab_url,omitempty"` - RemovableMediaVendor string `json:"removable_media_vendor,omitempty"` - RemovableMediaName string `json:"removable_media_name,omitempty"` - RemovableMediaSerialNumber string `json:"removable_media_serial_number,omitempty"` - RemovableMediaCapacity *int `json:"removable_media_capacity,omitempty"` - RemovableMediaBusType string `json:"removable_media_bus_type,omitempty"` - RemovableMediaMediaName string `json:"removable_media_media_name,omitempty"` - RemovableMediaVolumeName string `json:"removable_media_volume_name,omitempty"` - RemovableMediaPartitionId string `json:"removable_media_partition_id,omitempty"` - SyncDestination string `json:"sync_destination,omitempty"` - SyncDestinationUsername string `json:"sync_destination_username,omitempty"` - EmailDLPPolicyNames []string `json:"email_dlp_policy_names,omitempty"` - EmailDLPSubject string `json:"email_dlp_subject,omitempty"` - EmailDLPSender string `json:"email_dlp_sender,omitempty"` - EmailDLPFrom string `json:"email_dlp_from,omitempty"` - EmailDLPRecipients []string `json:"email_dlp_recipients,omitempty"` - OutsideActiveHours *bool `json:"outside_active_hours,omitempty"` - IdentifiedExtensionMIMEType string `json:"identified_extension_mime_type,omitempty"` - CurrentExtensionMIMEType string `json:"current_extension_mime_type,omitempty"` - SuspiciousFileTypeMismatch *bool `json:"suspicious_file_type_mismatch,omitempty"` - PrintJobName string `json:"print_job_name,omitempty"` - PrinterName string `json:"printer_name,omitempty"` - PrintedFilesBackupPath string `json:"printed_files_backup_path,omitempty"` - RemoteActivity string `json:"remote_activity,omitempty"` - Trusted *bool `json:"trusted,omitempty"` - LoggedInOperatingSystemUser string `json:"logged_in_operating_system_user,omitempty"` -} - type ElasticFileEvent struct { Event *Event `json:"event,omitempty"` Timestamp *time.Time `json:"@timestamp,omitempty"` diff --git a/ffsEvent/ffsFetcher.go b/ffsEvent/ffsFetcher.go index 01923a1..48c8e47 100644 --- a/ffsEvent/ffsFetcher.go +++ b/ffsEvent/ffsFetcher.go @@ -104,7 +104,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg if len(*fileEvents) > 0 { //remap ffsEvents to ElasticFFSEvent var elasticFFSEvents []eventOutput.ElasticFileEvent - var semiElasticFFSEvents []eventOutput.SemiElasticFFSEvent var remapWg sync.WaitGroup remapWg.Add(len(*fileEvents)) go func() { @@ -381,125 +380,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg elasticFFSEvents = append(elasticFFSEvents, *elasticFileEvent) remapWg.Done() - } else if query.EsStandardized != "" && strings.EqualFold(query.EsStandardized, "half") { - semiElasticFileEvent := eventOutput.SemiElasticFileEvent{ - EventId: ffsEvent.EventId, - EventType: ffsEvent.EventType, - EventTimestamp: ffsEvent.EventTimestamp, - InsertionTimestamp: ffsEvent.InsertionTimestamp, - FilePath: ffsEvent.FilePath, - FileName: ffsEvent.FileName, - FileType: ffsEvent.FileType, - FileCategory: ffsEvent.FileCategory, - IdentifiedExtensionCategory: ffsEvent.IdentifiedExtensionCategory, - CurrentExtensionCategory: ffsEvent.CurrentExtensionCategory, - FileSize: ffsEvent.FileSize, - FileOwner: ffsEvent.FileOwner, - Md5Checksum: ffsEvent.Md5Checksum, - Sha256Checksum: ffsEvent.Sha256Checksum, - CreatedTimestamp: ffsEvent.CreatedTimestamp, - ModifyTimestamp: ffsEvent.ModifyTimestamp, - DeviceUsername: ffsEvent.DeviceUsername, - DeviceUid: ffsEvent.DeviceUid, - UserUid: ffsEvent.UserUid, - OsHostname: ffsEvent.OsHostname, - DomainName: ffsEvent.DomainName, - PublicIpAddress: ffsEvent.PublicIpAddress, - PrivateIpAddresses: ffsEvent.PrivateIpAddresses, - Actor: ffsEvent.Actor, - DirectoryId: ffsEvent.DirectoryId, - Source: ffsEvent.Source, - Url: ffsEvent.Url, - Shared: ffsEvent.Shared, - SharedWith: ffsEvent.SharedWith, - SharingTypeAdded: ffsEvent.SharingTypeAdded, - CloudDriveId: ffsEvent.CloudDriveId, - DetectionSourceAlias: ffsEvent.DetectionSourceAlias, - FileId: ffsEvent.FileId, - Exposure: ffsEvent.Exposure, - ProcessOwner: ffsEvent.ProcessOwner, - ProcessName: ffsEvent.ProcessName, - TabWindowTitle: ffsEvent.TabWindowTitle, - TabUrl: ffsEvent.TabUrl, - RemovableMediaVendor: ffsEvent.RemovableMediaVendor, - RemovableMediaName: ffsEvent.RemovableMediaName, - RemovableMediaSerialNumber: ffsEvent.RemovableMediaSerialNumber, - RemovableMediaCapacity: ffsEvent.RemovableMediaCapacity, - RemovableMediaBusType: ffsEvent.RemovableMediaBusType, - RemovableMediaMediaName: ffsEvent.RemovableMediaMediaName, - RemovableMediaVolumeName: ffsEvent.RemovableMediaVolumeName, - RemovableMediaPartitionId: ffsEvent.RemovableMediaPartitionId, - SyncDestination: ffsEvent.SyncDestination, - SyncDestinationUsername: ffsEvent.SyncDestinationUsername, - EmailDLPPolicyNames: ffsEvent.EmailDLPPolicyNames, - EmailDLPSubject: ffsEvent.EmailDLPSubject, - EmailDLPSender: ffsEvent.EmailDLPSender, - EmailDLPFrom: ffsEvent.EmailDLPSender, - EmailDLPRecipients: ffsEvent.EmailDLPRecipients, - OutsideActiveHours: ffsEvent.OutsideActiveHours, - IdentifiedExtensionMIMEType: ffsEvent.IdentifiedExtensionMIMEType, - CurrentExtensionMIMEType: ffsEvent.CurrentExtensionMIMEType, - SuspiciousFileTypeMismatch: ffsEvent.SuspiciousFileTypeMismatch, - PrintJobName: ffsEvent.PrintJobName, - PrinterName: ffsEvent.PrinterName, - PrintedFilesBackupPath: ffsEvent.PrintedFilesBackupPath, - RemoteActivity: ffsEvent.RemoteActivity, - Trusted: ffsEvent.Trusted, - LoggedInOperatingSystemUser: ffsEvent.LoggedInOperatingSystemUser, - } - - var semiElasticFFSEvent eventOutput.SemiElasticFFSEvent - var geo *eventOutput.Geo - if location != nil { - geo = &eventOutput.Geo{ - Status: location.Status, - Message: location.Message, - Continent: location.Continent, - ContinentCode: location.ContinentCode, - Country: location.Country, - CountryCode: location.CountryCode, - Region: location.Region, - RegionName: location.RegionName, - City: location.City, - District: location.District, - ZIP: location.ZIP, - Lat: location.Lat, - Lon: location.Lon, - Timezone: location.Timezone, - Currency: location.Currency, - ISP: location.ISP, - Org: location.Org, - AS: location.AS, - ASName: location.ASName, - Reverse: location.Reverse, - Mobile: location.Mobile, - Proxy: location.Proxy, - Hosting: location.Hosting, - Query: location.Query, - } - - if (location.Lat != nil && *location.Lat != 0) && (location.Lon != nil && *location.Lon != 0) { - geo.Location = &eventOutput.Location{ - Lat: location.Lat, - Lon: location.Lon, - } - } else { - geo.Location = nil - } - } - - semiElasticFFSEvent = eventOutput.SemiElasticFFSEvent{ - FileEvent: semiElasticFileEvent, - } - - if location != nil && location.Status == "" { - semiElasticFFSEvent.Geo = nil - } else { - semiElasticFFSEvent.Geo = geo - } - - semiElasticFFSEvents = append(semiElasticFFSEvents, semiElasticFFSEvent) - remapWg.Done() } } }() @@ -513,8 +393,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg err = eventOutput.WriteEvents(ffsEvents, query) } else if query.EsStandardized == "full" { err = eventOutput.WriteEvents(elasticFFSEvents, query) - } else if query.EsStandardized == "half" { - err = eventOutput.WriteEvents(semiElasticFFSEvents, query) } if err != nil { @@ -584,15 +462,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg elasticWg.Done() } }() - } else if query.EsStandardized == "half" { - elasticWg.Add(len(semiElasticFFSEvents)) - go func() { - for _, elasticFileEvent := range semiElasticFFSEvents { - r := elastic.NewBulkIndexRequest().Index(indexName).Doc(elasticFileEvent) - processor.Add(r) - elasticWg.Done() - } - }() } elasticWg.Wait() @@ -638,25 +507,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, elasticFileEvent.Event.Created.Format(query.Elasticsearch.IndexTimeAppend)) } - requiredIndexMutex.RLock() - if _, found := requiredIndexTimestamps[indexTime]; !found { - requiredIndexTimestamps[indexTime] = nil - } - requiredIndexMutex.RUnlock() - elasticWg.Done() - } - }() - } else if query.EsStandardized == "half" { - elasticWg.Add(len(semiElasticFFSEvents)) - go func() { - for _, elasticFileEvent := range semiElasticFFSEvents { - var indexTime time.Time - if query.Elasticsearch.IndexTimeGen == "insertionTimestamp" { - indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, elasticFileEvent.FileEvent.EventTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) - } else { - indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, elasticFileEvent.FileEvent.EventTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) - } - requiredIndexMutex.RLock() if _, found := requiredIndexTimestamps[indexTime]; !found { requiredIndexTimestamps[indexTime] = nil @@ -743,22 +593,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg elasticWg.Done() } }() - } else if query.EsStandardized == "half" { - elasticWg.Add(len(semiElasticFFSEvents)) - go func() { - for _, elasticFileEvent := range semiElasticFFSEvents { - var indexTime time.Time - if query.Elasticsearch.IndexTimeGen == "insertionTimestamp" { - indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, elasticFileEvent.FileEvent.InsertionTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) - } else { - indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, elasticFileEvent.FileEvent.EventTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) - } - indexName := elasticsearch.BuildIndexNameWithTime(query.Elasticsearch, indexTime) - r := elastic.NewBulkIndexRequest().Index(indexName).Doc(elasticFileEvent) - processor.Add(r) - elasticWg.Done() - } - }() } elasticWg.Wait() @@ -837,37 +671,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg _, err = writer.Write(event) - if err != nil { - //TODO handle error - log.Println("error writing ffs event") - log.Println(string(event)) - panic(err) - } - _, err = writer.Write([]byte("\n")) - if err != nil { - //TODO handle error - log.Println("error writing ffs event") - log.Println(string(event)) - panic(err) - } - logstashWg.Done() - } - }() - } else if query.EsStandardized == "half" { - logstashWg.Add(len(semiElasticFFSEvents)) - go func() { - for _, elasticFileEvent := range semiElasticFFSEvents { - event, err := json.Marshal(elasticFileEvent) - - if err != nil { - //TODO handle error - log.Println("error marshaling ffs event") - log.Println(elasticFileEvent) - panic(err) - } - - _, err = writer.Write(event) - if err != nil { //TODO handle error log.Println("error writing ffs event") From 9d5f2f5cda2e54fab93055b3683fb3efcd3c9966 Mon Sep 17 00:00:00 2001 From: BenB196 <8277432+BenB196@users.noreply.github.com> Date: Tue, 6 Oct 2020 11:32:12 -0400 Subject: [PATCH 2/5] Updated crashplan-ffs-go-pkg --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 64447f9..1cc7b9f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/BenB196/crashplan-ffs-puller go 1.15 require ( - github.com/BenB196/crashplan-ffs-go-pkg v0.2.2 + github.com/BenB196/crashplan-ffs-go-pkg v0.2.3 github.com/BenB196/ip-api-go-pkg v0.0.9 github.com/VictoriaMetrics/fastcache v1.5.7 github.com/olivere/elastic/v7 v7.0.20 diff --git a/go.sum b/go.sum index 08460d0..5dfbb14 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/BenB196/crashplan-ffs-go-pkg v0.2.1 h1:QTbMQ6IMryZqDtHL9bYTCrrkQH2RdK github.com/BenB196/crashplan-ffs-go-pkg v0.2.1/go.mod h1:8ikK7PskHvmZupmN1uUnL3+z/CeIsa+7ibvfPuY34Dg= github.com/BenB196/crashplan-ffs-go-pkg v0.2.2 h1:m3KVthWXH8X6/tosfdb+kga1b9LVHcBFxqof05AW7fU= github.com/BenB196/crashplan-ffs-go-pkg v0.2.2/go.mod h1:8ikK7PskHvmZupmN1uUnL3+z/CeIsa+7ibvfPuY34Dg= +github.com/BenB196/crashplan-ffs-go-pkg v0.2.3 h1:zDl4a6I3+4AMzTnCsyU+zUmHdI1dffiPZpALVbooALE= +github.com/BenB196/crashplan-ffs-go-pkg v0.2.3/go.mod h1:8ikK7PskHvmZupmN1uUnL3+z/CeIsa+7ibvfPuY34Dg= github.com/BenB196/ip-api-go-pkg v0.0.3 h1:FWmM7FkhT1N55jd4jPW7W9LVOQrG89DLlrgwMb5fosw= github.com/BenB196/ip-api-go-pkg v0.0.3/go.mod h1:ccPdkBNnzf/uvuk7qXgEO06TCS/qILNJQP/KETQG4jU= github.com/BenB196/ip-api-go-pkg v0.0.4 h1:BbWELxooG6l2gaXQ4i4gm6NsyikdbhalF+TKg08gDPQ= From 5cfcae6204920531749ff4262593d96462651107 Mon Sep 17 00:00:00 2001 From: BenB196 <8277432+BenB196@users.noreply.github.com> Date: Tue, 6 Oct 2020 15:57:32 -0400 Subject: [PATCH 3/5] Made ecs output much more ecs compliant --- config/configReader.go | 4 +- eventOutput/fileHandler.go | 239 +++++++++++++++----------- ffsEvent/ffsFetcher.go | 340 +++++++++++++++++++------------------ 3 files changed, 317 insertions(+), 266 deletions(-) diff --git a/config/configReader.go b/config/configReader.go index e7847f9..8010df1 100644 --- a/config/configReader.go +++ b/config/configReader.go @@ -460,8 +460,8 @@ func validateConfigJson(fileBytes []byte) (*Config, error) { } //validate esStandardized - if query.EsStandardized != "" && !strings.EqualFold(query.EsStandardized, "full") && !strings.EqualFold(query.EsStandardized, "half") { - panic("unknown value for esStandardized, values can either be full, half, or \"\"") + if query.EsStandardized != "" && !strings.EqualFold(query.EsStandardized, "ecs") { + panic("unknown value for esStandardized, values can either be ecs or \"\"") } //Validate ip-api diff --git a/eventOutput/fileHandler.go b/eventOutput/fileHandler.go index 27d5391..dae8970 100644 --- a/eventOutput/fileHandler.go +++ b/eventOutput/fileHandler.go @@ -20,123 +20,74 @@ type FFSEvent struct { GeoLocation *Location `json:"geoPoint,omitempty"` } -type ElasticFileEvent struct { - Event *Event `json:"event,omitempty"` - Timestamp *time.Time `json:"@timestamp,omitempty"` - File *File `json:"file,omitempty"` - User *User `json:"user,omitempty"` - Host *Host `json:"host,omitempty"` - Client *Client `json:"client,omitempty"` - Process *Process `json:"process,omitempty"` - Tab *Tab `json:"tab,omitempty"` - RemovableMedia *RemovableMedia `json:"removable_media,omitempty"` - EmailDlp *EmailDlp `json:"email_dlp,omitempty"` - Printing *Printing `json:"printing,omitempty"` -} - -type Event struct { - Id string `json:"id,omitempty"` - Type string `json:"type,omitempty"` - Ingested *time.Time `json:"ingested,omitempty"` - Created *time.Time `json:"created,omitempty"` - Module string `json:"module,omitempty"` - Dataset []string `json:"dataset,omitempty"` - OutsideActiveHours *bool `json:"outside_active_hours,omitempty"` +type Code42 struct { + Event *Code42Event `json:"event,omitempty"` + InsertionTimestamp *time.Time `json:"insertion_timestamp,omitempty"` + File *Code42File `json:"file,omitempty"` + Device *Code42Device `json:"device,omitempty"` + OsHostName string `json:"os_host_name,omitempty"` + DomainName string `json:"domain_name,omitempty"` + PublicIpAddress string `json:"public_ip_address,omitempty"` + PrivateIpAddresses []string `json:"private_ip_addresses,omitempty"` + Actor string `json:"actor,omitempty"` + DirectoryId []string `json:"directory_id,omitempty"` + Source string `json:"source,omitempty"` + Url *URL `json:"url,omitempty"` + Shared *bool `json:"shared,omitempty"` + SharedWith []string `json:"shared_with,omitempty"` + SharingTypeAdded []string `json:"sharing_type_added,omitempty"` + CloudDriveId string `json:"cloud_drive_id,omitempty"` + DetectionSourceAlias string `json:"detection_source_alias,omitempty"` + Exposure []string `json:"exposure,omitempty"` + Process *Process `json:"process,omitempty"` + Tab *Code42Tab `json:"tab,omitempty"` + RemovableMedia *Code42RemovableMedia `json:"removable_media,omitempty"` + SyncDestination string `json:"sync_destination,omitempty"` + SyncDestinationUsername string `json:"sync_destination_username,omitempty"` + EmailDlp *Code42EmailDlp `json:"email_dlp,omitempty"` + OutsideActiveHours *bool `json:"outside_active_hours,omitempty"` + Print *Code42Print `json:"print,omitempty"` + RemoteActivity string `json:"remote_activity,omitempty"` + Trusted *bool `json:"trusted,omitempty"` + LoggedInOperatingSystemUser string `json:"logged_in_operating_system_user,omitempty"` + Destination *Code42Destination `json:"destination,omitempty"` } -type Hash struct { - Md5 string `json:"md5,omitempty"` - Sha256 string `json:"sha256,omitempty"` +type Code42Event struct { + Id string `json:"id,omitempty"` + Type string `json:"type,omitempty"` + Timestamp *time.Time `json:"timestamp,omitempty"` } -type URL struct { - Full string `json:"full,omitempty"` - Domain string `json:"domain,omitempty"` - Extension string `json:"extension,omitempty"` - Fragment string `json:"fragment,omitempty"` - Path string `json:"path,omitempty"` - Port *int `json:"port,omitempty"` - Query string `json:"query,omitempty"` - Scheme string `json:"scheme,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - RegisteredDomain string `json:"registered_domain,omitempty"` - TopLevelDomain string `json:"top_level_domain,omitempty"` -} - -type File struct { +type Code42File struct { Path string `json:"path,omitempty"` Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` Category string `json:"category,omitempty"` IdentifiedExtensionCategory string `json:"identified_extension_category,omitempty"` CurrentExtensionCategory string `json:"current_extension_category,omitempty"` - Extension []string `json:"extension,omitempty"` //Array of extensions Size *int `json:"size,omitempty"` - Owner []string `json:"owner,omitempty"` //Array of owners + Owner []string `json:"owner,omitempty"` Hash *Hash `json:"hash,omitempty"` - Created *time.Time `json:"created,omitempty"` - Mtime *time.Time `json:"mtime,omitempty"` - Directory []string `json:"directory,omitempty"` - URL *URL `json:"url,omitempty"` - Shared *bool `json:"shared,omitempty"` - SharedWith []string `json:"shared_with,omitempty"` - SharingTypeAdded []string `json:"sharing_type_added,omitempty"` - CloudDriveId string `json:"cloud_drive_id,omitempty"` - DetectionSourceAlias string `json:"detection_source_alias,omitempty"` - SyncDestination string `json:"sync_destination,omitempty"` - SyncDestinationUser *User `json:"sync_destination_user,omitempty"` + CreatedTimestamp *time.Time `json:"created_timestamp,omitempty"` + ModifyTimestamp *time.Time `json:"modify_timestamp,omitempty"` Id string `json:"id,omitempty"` IdentifiedExtensionMIMEType string `json:"identified_extension_mime_type,omitempty"` CurrentExtensionMIMEType string `json:"current_extension_mime_type,omitempty"` SuspiciousFileTypeMismatch *bool `json:"suspicious_file_type_mismatch,omitempty"` - RemoteActivity string `json:"remote_activity,omitempty"` - Trusted *bool `json:"trusted,omitempty"` } -type User struct { - Email string `json:"email,omitempty"` - Id string `json:"id,omitempty"` - Actor string `json:"actor,omitempty"` -} - -type Host struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Hostname string `json:"hostname,omitempty"` - User *User `json:"user,omitempty"` -} - -type Nat struct { - Ip []string `json:"ip,omitempty"` -} - -type Organization struct { - Name string `json:"name,omitempty"` +type Code42Device struct { + Username string `json:"username,omitempty"` + Uid string `json:"uid,omitempty"` } -type AS struct { - Organization *Organization `json:"organization,omitempty"` -} - -type Client struct { - Ip string `json:"ip,omitempty"` - Nat *Nat `json:"nat,omitempty"` - Geo *Geo `json:"geo,omitempty"` - AS *AS `json:"as,omitempty"` -} - -type Process struct { - ProcessOwner string `json:"owner,omitempty"` - ProcessName string `json:"name,omitempty"` -} - -type Tab struct { +type Code42Tab struct { WindowTitle string `json:"window_title,omitempty"` - URL *URL `json:"url,omitempty"` + Url *URL `json:"url,omitempty"` } -type RemovableMedia struct { +type Code42RemovableMedia struct { Vendor string `json:"vendor,omitempty"` Name string `json:"name,omitempty"` SerialNumber string `json:"serial_number,omitempty"` @@ -147,7 +98,7 @@ type RemovableMedia struct { PartitionId string `json:"partition_id,omitempty"` } -type EmailDlp struct { +type Code42EmailDlp struct { PolicyNames []string `json:"policy_names,omitempty"` Subject string `json:"subject,omitempty"` Sender string `json:"sender,omitempty"` @@ -155,16 +106,104 @@ type EmailDlp struct { Recipients []string `json:"recipients,omitempty"` } -type Printing struct { - JobName string `json:"job_name,omitempty"` - Printer *Printer `json:"printer,omitempty"` - PrintedFilesBackupPath string `json:"printed_files_backup_path,omitempty"` +type Code42Print struct { + JobName string `json:"job_name,omitempty"` + PrinterName string `json:"name,omitempty"` + PrintedFilesBackupPath string `json:"printed_files_backup_path,omitempty"` } -type Printer struct { +type Code42Destination struct { + Category string `json:"category,omitempty"` + Name string `json:"name,omitempty"` +} + +type ElasticFileEvent struct { + Event *Event `json:"event,omitempty"` + Timestamp *time.Time `json:"@timestamp,omitempty"` + File *File `json:"file,omitempty"` + Host *Host `json:"host,omitempty"` + Process *Process `json:"process,omitempty"` + Code42 *Code42 `json:"code_42,omitempty"` +} + +type Event struct { + Action string `json:"action,omitempty"` + Category string `json:"category,omitempty"` + Created *time.Time `json:"created,omitempty"` + Dataset string `json:"dataset,omitempty"` + Id string `json:"id,omitempty"` + Ingested *time.Time `json:"ingested,omitempty"` + Kind string `json:"kind,omitempty"` + Module string `json:"module,omitempty"` + Outcome string `json:"outcome,omitempty"` + Provider string `json:"provider,omitempty"` + Type string `json:"type,omitempty"` +} + +type Hash struct { + Md5 string `json:"md5,omitempty"` + Sha256 string `json:"sha256,omitempty"` +} + +type URL struct { + Full string `json:"full,omitempty"` + Domain string `json:"domain,omitempty"` + Extension string `json:"extension,omitempty"` + Fragment string `json:"fragment,omitempty"` + Path string `json:"path,omitempty"` + Port *int `json:"port,omitempty"` + Query string `json:"query,omitempty"` + Scheme string `json:"scheme,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + RegisteredDomain string `json:"registered_domain,omitempty"` + TopLevelDomain string `json:"top_level_domain,omitempty"` +} + +type File struct { + Created *time.Time `json:"created,omitempty"` + Directory []string `json:"directory,omitempty"` + Extension string `json:"extension,omitempty"` + MimeType string `json:"mime_type,omitempty"` + Mtime *time.Time `json:"mtime,omitempty"` + Name string `json:"name,omitempty"` + Owner []string `json:"owner,omitempty"` + Path string `json:"path,omitempty"` + Size *int `json:"size,omitempty"` + Type string `json:"type,omitempty"` + Hash *Hash `json:"hash,omitempty"` +} + +type User struct { + Email string `json:"email,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Domain string `json:"domain,omitempty"` +} + +type Organization struct { Name string `json:"name,omitempty"` } +type AS struct { + Organization *Organization `json:"organization,omitempty"` +} + +type Host struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Hostname string `json:"hostname,omitempty"` + User *User `json:"user,omitempty"` + IP []string `json:"ip,omitempty"` + Geo *Geo `json:"geo,omitempty"` + AS *AS `json:"as,omitempty"` +} + +type Process struct { + Owner string `json:"owner,omitempty"` + Name string `json:"name,omitempty"` +} + type Geo struct { Status string `json:"status,omitempty"` Message string `json:"message,omitempty"` diff --git a/ffsEvent/ffsFetcher.go b/ffsEvent/ffsFetcher.go index 48c8e47..3523766 100644 --- a/ffsEvent/ffsFetcher.go +++ b/ffsEvent/ffsFetcher.go @@ -12,6 +12,7 @@ import ( ip_api "github.com/BenB196/ip-api-go-pkg" "github.com/olivere/elastic/v7" "log" + "path/filepath" "strconv" "strings" "sync" @@ -121,29 +122,34 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg } } - if query.EsStandardized != "" && strings.EqualFold(query.EsStandardized, "full") { - event := &eventOutput.Event{ - Id: ffsEvent.EventId, - Type: ffsEvent.EventType, - Ingested: ffsEvent.InsertionTimestamp, - Created: ffsEvent.EventTimestamp, - Module: ffsEvent.Source, - Dataset: ffsEvent.Exposure, - OutsideActiveHours: ffsEvent.OutsideActiveHours, - } - - timestamp := ffsEvent.EventTimestamp - - var extensions []string + if query.EsStandardized != "" && strings.EqualFold(query.EsStandardized, "ecs") { - if ffsEvent.IdentifiedExtensionCategory != "" { - extensions = append(extensions, ffsEvent.IdentifiedExtensionCategory) + //Event processing + eventType := "info" + if strings.EqualFold(ffsEvent.EventType, "created") { + eventType = "creation" + } else if strings.EqualFold(ffsEvent.EventType, "modified") { + eventType = "change" + } else if strings.EqualFold(ffsEvent.EventType, "deleted") { + eventType = "deletion" } - if ffsEvent.CurrentExtensionCategory != "" { - extensions = append(extensions, ffsEvent.CurrentExtensionCategory) - } + event := &eventOutput.Event{ + Action: ffsEvent.EventType, + Category: "file", + Created: ffsEvent.EventTimestamp, + Dataset: "code42.ffs", + Id: ffsEvent.EventId, + Ingested: ffsEvent.InsertionTimestamp, + Kind: "event", + Module: "code42", + Type: eventType, + } + + //@timestamp + timestamp := ffsEvent.EventTimestamp + //file fields hash := &eventOutput.Hash{ Md5: ffsEvent.Md5Checksum, Sha256: ffsEvent.Sha256Checksum, @@ -153,92 +159,72 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg hash = nil } - url := getUrlInfo(ffsEvent.Url) - - var syncDestinationUser *eventOutput.User - - if ffsEvent.SyncDestinationUsername != "" && ffsEvent.SyncDestinationUsername != "NAME_NOT_AVAILABLE" { - syncDestinationUser = &eventOutput.User{ - Id: ffsEvent.SyncDestinationUsername, - } + fileType := "unknown" + if strings.EqualFold(ffsEvent.FileType, "file") || strings.EqualFold(ffsEvent.FileType, "win_nds") || strings.EqualFold(ffsEvent.FileType, "mac_rsrc") || strings.EqualFold(ffsEvent.FileType, "fifo") || strings.EqualFold(ffsEvent.FileType, "bundle") { + fileType = "file" + } else if strings.EqualFold(ffsEvent.FileType, "dir") || strings.EqualFold(ffsEvent.FileType, "block_device") || strings.EqualFold(ffsEvent.FileType, "char_device") { + fileType = "dir" + } else if strings.EqualFold(ffsEvent.FileType, "symlink") { + fileType = "symlink" } file := &eventOutput.File{ - Path: ffsEvent.FilePath, - Name: ffsEvent.FileName, - Type: ffsEvent.FileType, - Category: ffsEvent.FileCategory, - IdentifiedExtensionCategory: ffsEvent.IdentifiedExtensionCategory, - CurrentExtensionCategory: ffsEvent.CurrentExtensionCategory, - Extension: extensions, - Size: ffsEvent.FileSize, - Owner: ffsEvent.FileOwner, - Hash: hash, - Created: ffsEvent.CreatedTimestamp, - Mtime: ffsEvent.ModifyTimestamp, - Directory: ffsEvent.DirectoryId, - URL: url, - Shared: ffsEvent.Shared, - SharedWith: ffsEvent.SharedWith, - SharingTypeAdded: ffsEvent.SharingTypeAdded, - CloudDriveId: ffsEvent.CloudDriveId, - DetectionSourceAlias: ffsEvent.DetectionSourceAlias, - SyncDestination: ffsEvent.SyncDestination, - SyncDestinationUser: syncDestinationUser, - Id: ffsEvent.FileId, - IdentifiedExtensionMIMEType: ffsEvent.IdentifiedExtensionMIMEType, - CurrentExtensionMIMEType: ffsEvent.CurrentExtensionMIMEType, - SuspiciousFileTypeMismatch: ffsEvent.SuspiciousFileTypeMismatch, - RemoteActivity: ffsEvent.RemoteActivity, - Trusted: ffsEvent.Trusted, - } - + Path: ffsEvent.FilePath, + Name: ffsEvent.FileName, + Type: fileType, + Extension: filepath.Ext(ffsEvent.FileName), + Size: ffsEvent.FileSize, + Owner: ffsEvent.FileOwner, + Hash: hash, + Created: ffsEvent.CreatedTimestamp, + Mtime: ffsEvent.ModifyTimestamp, + Directory: ffsEvent.DirectoryId, + MimeType: ffsEvent.CurrentExtensionMIMEType, + } + + //user fields var user *eventOutput.User if ffsEvent.DeviceUsername == "NAME_NOT_AVAILABLE" { + name := "" + domain := "" + if strings.Contains(ffsEvent.Actor, "@") { + name = strings.Split(ffsEvent.Actor, "@")[0] + domain = strings.Split(ffsEvent.DeviceUsername, "@")[1] + } + user = &eventOutput.User{ - Email: ffsEvent.Actor, - Id: ffsEvent.UserUid, - Actor: ffsEvent.Actor, + Email: ffsEvent.Actor, + Id: ffsEvent.UserUid, + Name: name, + Domain: domain, } } else { + name := "" + domain := "" + if strings.Contains(ffsEvent.DeviceUsername, "@") { + name = strings.Split(ffsEvent.DeviceUsername, "@")[0] + domain = strings.Split(ffsEvent.DeviceUsername, "@")[1] + } user = &eventOutput.User{ - Email: ffsEvent.DeviceUsername, - Id: ffsEvent.UserUid, - Actor: ffsEvent.Actor, + Email: ffsEvent.DeviceUsername, + Id: ffsEvent.UserUid, + Name: name, + Domain: domain, } } - if *user == (eventOutput.User{}) { - user = nil - } - - var hostUser *eventOutput.User - if ffsEvent.LoggedInOperatingSystemUser != "" && ffsEvent.LoggedInOperatingSystemUser != "NAME_NOT_AVAILABLE" { - hostUser = &eventOutput.User{ - Id: ffsEvent.LoggedInOperatingSystemUser, - } - } - - host := &eventOutput.Host{ - Id: ffsEvent.DeviceUid, - Name: ffsEvent.OsHostname, - Hostname: ffsEvent.DomainName, - User: hostUser, + user.Id = ffsEvent.LoggedInOperatingSystemUser } - if *host == (eventOutput.Host{}) { - host = nil + if *user == (eventOutput.User{}) { + user = nil } - var nat *eventOutput.Nat - - if ffsEvent.PrivateIpAddresses != nil { - nat = &eventOutput.Nat{Ip: ffsEvent.PrivateIpAddresses} - } else { - nat = nil - } + //host ips + ips := ffsEvent.PrivateIpAddresses + ips = append(ips, ffsEvent.PublicIpAddress) var geo *eventOutput.Geo var as *eventOutput.AS @@ -288,94 +274,120 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg geo = nil } - client := &eventOutput.Client{ - Ip: ffsEvent.PublicIpAddress, - Nat: nat, - Geo: geo, - AS: as, - } - - if *client == (eventOutput.Client{}) { - client = nil + host := &eventOutput.Host{ + Id: ffsEvent.DeviceUid, + Name: ffsEvent.OsHostname, + Hostname: ffsEvent.DomainName, + User: user, + IP: ips, + Geo: geo, + AS: as, } process := &eventOutput.Process{ - ProcessOwner: ffsEvent.ProcessOwner, - ProcessName: ffsEvent.ProcessName, + Owner: ffsEvent.ProcessOwner, + Name: ffsEvent.ProcessName, } if *process == (eventOutput.Process{}) { process = nil } - tabUrl := getUrlInfo(ffsEvent.TabUrl) - - tab := &eventOutput.Tab{ - WindowTitle: ffsEvent.TabWindowTitle, - URL: tabUrl, - } - - if *tab == (eventOutput.Tab{}) { - tab = nil - } - - removableMedia := &eventOutput.RemovableMedia{ - Vendor: ffsEvent.RemovableMediaVendor, - Name: ffsEvent.RemovableMediaName, - SerialNumber: ffsEvent.RemovableMediaSerialNumber, - Capacity: ffsEvent.RemovableMediaCapacity, - BusType: ffsEvent.RemovableMediaBusType, - MediaName: ffsEvent.RemovableMediaMediaName, - VolumeName: ffsEvent.RemovableMediaVolumeName, - PartitionId: ffsEvent.RemovableMediaPartitionId, + //code 42 fields + code42Event := &eventOutput.Code42Event{ + Id: ffsEvent.EventId, + Type: ffsEvent.EventType, + Timestamp: ffsEvent.EventTimestamp, } - if *removableMedia == (eventOutput.RemovableMedia{}) { - removableMedia = nil - } - - emailDlp := &eventOutput.EmailDlp{ - PolicyNames: ffsEvent.EmailDLPPolicyNames, - Subject: ffsEvent.EmailDLPSubject, - Sender: ffsEvent.EmailDLPSender, - From: ffsEvent.EmailDLPFrom, - Recipients: ffsEvent.EmailDLPRecipients, - } - - if ffsEvent.EmailDLPPolicyNames == nil && ffsEvent.EmailDLPSubject == "" && ffsEvent.EmailDLPSender == "" && ffsEvent.EmailDLPFrom == "" && ffsEvent.EmailDLPRecipients == nil { - emailDlp = nil - } - - printer := &eventOutput.Printer{ - Name: ffsEvent.PrinterName, - } - - if ffsEvent.PrinterName == "" { - printer = nil - } - - printing := &eventOutput.Printing{ - JobName: ffsEvent.PrintJobName, - Printer: printer, - PrintedFilesBackupPath: ffsEvent.PrintedFilesBackupPath, + code42File := &eventOutput.Code42File{ + Path: ffsEvent.FilePath, + Name: ffsEvent.FileName, + Type: ffsEvent.FileType, + Category: ffsEvent.FileCategory, + IdentifiedExtensionCategory: ffsEvent.IdentifiedExtensionCategory, + CurrentExtensionCategory: ffsEvent.CurrentExtensionCategory, + Size: ffsEvent.FileSize, + Owner: ffsEvent.FileOwner, + Hash: hash, + CreatedTimestamp: ffsEvent.CreatedTimestamp, + ModifyTimestamp: ffsEvent.ModifyTimestamp, + Id: ffsEvent.FileId, + IdentifiedExtensionMIMEType: ffsEvent.IdentifiedExtensionMIMEType, + CurrentExtensionMIMEType: ffsEvent.CurrentExtensionMIMEType, + SuspiciousFileTypeMismatch: ffsEvent.SuspiciousFileTypeMismatch, } - if *printing == (eventOutput.Printing{}) { - printing = nil + code42Device := &eventOutput.Code42Device{ + Username: ffsEvent.DeviceUsername, + Uid: ffsEvent.DeviceUid, + } + + code42 := &eventOutput.Code42{ + Event: code42Event, + InsertionTimestamp: ffsEvent.InsertionTimestamp, + File: code42File, + Device: code42Device, + OsHostName: ffsEvent.OsHostname, + DomainName: ffsEvent.DomainName, + PublicIpAddress: ffsEvent.PublicIpAddress, + PrivateIpAddresses: ffsEvent.PrivateIpAddresses, + Actor: ffsEvent.Actor, + DirectoryId: ffsEvent.DirectoryId, + Source: ffsEvent.Source, + Url: getUrlInfo(ffsEvent.Url), + Shared: ffsEvent.Shared, + SharedWith: ffsEvent.SharedWith, + SharingTypeAdded: ffsEvent.SharingTypeAdded, + CloudDriveId: ffsEvent.CloudDriveId, + DetectionSourceAlias: ffsEvent.DetectionSourceAlias, + Exposure: ffsEvent.Exposure, + Process: process, + Tab: &eventOutput.Code42Tab{ + WindowTitle: ffsEvent.TabWindowTitle, + Url: getUrlInfo(ffsEvent.TabUrl), + }, + RemovableMedia: &eventOutput.Code42RemovableMedia{ + Vendor: ffsEvent.RemovableMediaVendor, + Name: ffsEvent.RemovableMediaName, + SerialNumber: ffsEvent.RemovableMediaSerialNumber, + Capacity: ffsEvent.RemovableMediaCapacity, + BusType: ffsEvent.RemovableMediaBusType, + MediaName: ffsEvent.RemovableMediaMediaName, + VolumeName: ffsEvent.RemovableMediaVolumeName, + PartitionId: ffsEvent.RemovableMediaPartitionId, + }, + SyncDestination: ffsEvent.SyncDestination, + SyncDestinationUsername: ffsEvent.SyncDestinationUsername, + EmailDlp: &eventOutput.Code42EmailDlp{ + PolicyNames: ffsEvent.EmailDLPPolicyNames, + Subject: ffsEvent.EmailDLPSubject, + Sender: ffsEvent.EmailDLPSender, + From: ffsEvent.EmailDLPFrom, + Recipients: ffsEvent.EmailDLPRecipients, + }, + OutsideActiveHours: ffsEvent.OutsideActiveHours, + Print: &eventOutput.Code42Print{ + JobName: ffsEvent.PrintJobName, + PrinterName: ffsEvent.PrinterName, + PrintedFilesBackupPath: ffsEvent.PrintedFilesBackupPath, + }, + RemoteActivity: ffsEvent.RemoteActivity, + Trusted: ffsEvent.Trusted, + LoggedInOperatingSystemUser: ffsEvent.LoggedInOperatingSystemUser, + Destination: &eventOutput.Code42Destination{ + Category: ffsEvent.DestinationCategory, + Name: ffsEvent.DetectionSourceAlias, + }, } elasticFileEvent := &eventOutput.ElasticFileEvent{ - Event: event, - Timestamp: timestamp, - File: file, - User: user, - Host: host, - Client: client, - Process: process, - Tab: tab, - RemovableMedia: removableMedia, - EmailDlp: emailDlp, - Printing: printing, + Event: event, + Timestamp: timestamp, + File: file, + Host: host, + Process: process, + Code42: code42, } elasticFFSEvents = append(elasticFFSEvents, *elasticFileEvent) @@ -391,7 +403,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg case "file": if query.EsStandardized == "" { err = eventOutput.WriteEvents(ffsEvents, query) - } else if query.EsStandardized == "full" { + } else if query.EsStandardized == "ecs" { err = eventOutput.WriteEvents(elasticFFSEvents, query) } @@ -453,7 +465,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg elasticWg.Done() } }() - } else if query.EsStandardized == "full" { + } else if query.EsStandardized == "ecs" { elasticWg.Add(len(elasticFFSEvents)) go func() { for _, elasticFileEvent := range elasticFFSEvents { @@ -496,7 +508,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg elasticWg.Done() } }() - } else if query.EsStandardized == "full" { + } else if query.EsStandardized == "ecs" { elasticWg.Add(len(elasticFFSEvents)) go func() { for _, elasticFileEvent := range elasticFFSEvents { @@ -577,7 +589,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg elasticWg.Done() } }() - } else if query.EsStandardized == "full" { + } else if query.EsStandardized == "ecs" { elasticWg.Add(len(elasticFFSEvents)) go func() { for _, elasticFileEvent := range elasticFFSEvents { @@ -656,7 +668,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg logstashWg.Done() } }() - } else if query.EsStandardized == "full" { + } else if query.EsStandardized == "ecs" { logstashWg.Add(len(elasticFFSEvents)) go func() { for _, elasticFileEvent := range elasticFFSEvents { From 1e50f63195d888ff67e4e1ed5519b1fdd7f63daa Mon Sep 17 00:00:00 2001 From: BenB196 <8277432+BenB196@users.noreply.github.com> Date: Tue, 6 Oct 2020 15:58:05 -0400 Subject: [PATCH 4/5] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a2268e2..60a2d3e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 \ No newline at end of file +0.4.0 \ No newline at end of file From 945b20d7965e77b412b0067b53d146cd884021a7 Mon Sep 17 00:00:00 2001 From: BenB196 <8277432+BenB196@users.noreply.github.com> Date: Tue, 6 Oct 2020 16:32:34 -0400 Subject: [PATCH 5/5] Moved process to code 42 only --- eventOutput/fileHandler.go | 14 ++------------ ffsEvent/ffsFetcher.go | 13 ++----------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/eventOutput/fileHandler.go b/eventOutput/fileHandler.go index dae8970..6ec87a3 100644 --- a/eventOutput/fileHandler.go +++ b/eventOutput/fileHandler.go @@ -39,7 +39,7 @@ type Code42 struct { CloudDriveId string `json:"cloud_drive_id,omitempty"` DetectionSourceAlias string `json:"detection_source_alias,omitempty"` Exposure []string `json:"exposure,omitempty"` - Process *Process `json:"process,omitempty"` + Process *Code42Process `json:"process,omitempty"` Tab *Code42Tab `json:"tab,omitempty"` RemovableMedia *Code42RemovableMedia `json:"removable_media,omitempty"` SyncDestination string `json:"sync_destination,omitempty"` @@ -122,7 +122,6 @@ type ElasticFileEvent struct { Timestamp *time.Time `json:"@timestamp,omitempty"` File *File `json:"file,omitempty"` Host *Host `json:"host,omitempty"` - Process *Process `json:"process,omitempty"` Code42 *Code42 `json:"code_42,omitempty"` } @@ -181,14 +180,6 @@ type User struct { Domain string `json:"domain,omitempty"` } -type Organization struct { - Name string `json:"name,omitempty"` -} - -type AS struct { - Organization *Organization `json:"organization,omitempty"` -} - type Host struct { Id string `json:"id,omitempty"` Name string `json:"name,omitempty"` @@ -196,10 +187,9 @@ type Host struct { User *User `json:"user,omitempty"` IP []string `json:"ip,omitempty"` Geo *Geo `json:"geo,omitempty"` - AS *AS `json:"as,omitempty"` } -type Process struct { +type Code42Process struct { Owner string `json:"owner,omitempty"` Name string `json:"name,omitempty"` } diff --git a/ffsEvent/ffsFetcher.go b/ffsEvent/ffsFetcher.go index 3523766..e7863e1 100644 --- a/ffsEvent/ffsFetcher.go +++ b/ffsEvent/ffsFetcher.go @@ -227,7 +227,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg ips = append(ips, ffsEvent.PublicIpAddress) var geo *eventOutput.Geo - var as *eventOutput.AS if location != nil { geo = &eventOutput.Geo{ Status: location.Status, @@ -264,12 +263,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg } else { geo.Location = nil } - - if geo.ISP != "" { - as = &eventOutput.AS{Organization: &eventOutput.Organization{Name: geo.ISP}} - } else { - as = nil - } } else { geo = nil } @@ -281,15 +274,14 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg User: user, IP: ips, Geo: geo, - AS: as, } - process := &eventOutput.Process{ + process := &eventOutput.Code42Process{ Owner: ffsEvent.ProcessOwner, Name: ffsEvent.ProcessName, } - if *process == (eventOutput.Process{}) { + if *process == (eventOutput.Code42Process{}) { process = nil } @@ -386,7 +378,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg Timestamp: timestamp, File: file, Host: host, - Process: process, Code42: code42, }