Skip to content

Commit

Permalink
Merge pull request #27 from BenB196/staging
Browse files Browse the repository at this point in the history
Merge staging into master and bump to v0.0.9
  • Loading branch information
BenB196 authored Sep 27, 2019
2 parents f712fde + 12ff270 commit ff567e4
Show file tree
Hide file tree
Showing 7 changed files with 476 additions and 219 deletions.
12 changes: 10 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,9 @@ Currently, only JSON formatted configuration files are accepted, in the future Y
"numberOfShards": 1, #The number of shards the index should be created with
"numberOfReplicas": 0, #The number of replicas the index should be created with
"indexName": "crashplan", #The index name
"indexTimeAppend": "2006-01-02", #If you want to append a time format to the index name do it here. Must match the Golang time format pattern (This example is yyyy-MM-dd)
"indexTimeGen": "onOrBefore", #How to determine what time to use for the time stamp. Supports timeNow, onOrBefore, eventTimestamp, or insertionTimestamp.
"indexTimeAppend": "2006-01-02", #If you want to append a time format to the index name do it here. Must match the Golang time format pattern (This example is yyyy-MM-dd). Default: 2006-01-02
"indexTimeGen": "onOrBefore", #How to determine what time to use for the time stamp. Supports timeNow, onOrBefore, eventTimestamp, or insertionTimestamp. Default: timeNow
"useCustomIndexPattern": false #This allows you to use a custom Elasticsearch Index Template instead of using the build in Elasticsearch Index Pattern provided by the application. Default: false
"elasticUrl": "http://elasticsearch:9200", #The elasticsearch URL
"sniffing": false, #This determines whether the application will automatically try update its elasticsearch node list
"bestCompression": false, #This allows for indexes to be created with best_compression codec enabled
Expand Down Expand Up @@ -197,6 +198,13 @@ If you are using the elastic output type there are a few important things to und
1. onOrBefore, this will look at the onOrBefore time of the just completed query and set the appended value based off of it (this is useful if you are querying either old or new data, as it will spread the old data out over more indexes).
1. eventTimestamp, this will look at the eventTimestamp of the event and set the index name based off of it
1. insertTimestamp, this will look at the insertTimestamp of the event and set the index name based off of it.
1. If useCustomIndexPattern is set to true then you must set an Index Template up before proceeding. A basic index template can be found [here](docs/default_index_template.json).
1. If useCustomIndexPattern is set to false the following Elasticsearch configuration settings are ignored:
1. numberOfShards
1. numberOfReplicas
1. bestCompression
1. refreshInterval
1. aliases

### IP-API Integration

Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.0.8
0.0.9
29 changes: 15 additions & 14 deletions config/configReader.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,18 @@ type IPAPI struct {
}

type Elasticsearch struct {
NumberOfShards int `json:"numberOfShards,omitempty"`
NumberOfReplicas int `json:"numberOfReplicas,omitempty"`
IndexName string `json:"indexName,omitempty"`
IndexTimeAppend string `json:"indexTimeAppend,omitempty"`
IndexTimeGen string `json:"indexTimeGen,omitempty"`
ElasticURL string `json:"elasticUrl,omitempty"`
BasicAuth BasicAuth `json:"basicAuth,omitempty"`
Sniffing bool `json:"sniffing,omitempty"`
BestCompression bool `json:"bestCompression,omitempty"`
RefreshInterval int `json:"refreshInterval,omitempty"`
Aliases []string `json:"aliases,omitempty"`
NumberOfShards int `json:"numberOfShards,omitempty"`
NumberOfReplicas int `json:"numberOfReplicas,omitempty"`
IndexName string `json:"indexName,omitempty"`
IndexTimeAppend string `json:"indexTimeAppend,omitempty"`
IndexTimeGen string `json:"indexTimeGen,omitempty"`
ElasticURL string `json:"elasticUrl,omitempty"`
UseCustomIndexPattern bool `json:"useCustomIndexPattern"`
BasicAuth BasicAuth `json:"basicAuth,omitempty"`
Sniffing bool `json:"sniffing,omitempty"`
BestCompression bool `json:"bestCompression,omitempty"`
RefreshInterval int `json:"refreshInterval,omitempty"`
Aliases []string `json:"aliases,omitempty"`
}

type BasicAuth struct {
Expand Down Expand Up @@ -320,12 +321,12 @@ func validateConfigJson(fileBytes []byte) (Config, error) {
}

//validate number of shards
if query.Elasticsearch.NumberOfShards < 1 {
if !query.Elasticsearch.UseCustomIndexPattern && query.Elasticsearch.NumberOfShards < 1 {
return config, errors.New("error: number of shards for ffs query: " + query.Name + " cannot be lower than 1")
}

//validate number of replicas
if query.Elasticsearch.NumberOfReplicas < 0 {
if !query.Elasticsearch.UseCustomIndexPattern && query.Elasticsearch.NumberOfReplicas < 0 {
return config, errors.New("error: number of shards for ffs query: " + query.Name + " cannot be lower than 0")
}

Expand Down Expand Up @@ -366,7 +367,7 @@ func validateConfigJson(fileBytes []byte) (Config, error) {
}

//validate aliases
if len(query.Elasticsearch.Aliases) > 0 {
if !query.Elasticsearch.UseCustomIndexPattern && len(query.Elasticsearch.Aliases) > 0 {
for _, alias := range query.Elasticsearch.Aliases {
//validate alias names
err = utils.ValidateIndexName(alias)
Expand Down
232 changes: 232 additions & 0 deletions docs/default_index_template.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,232 @@
{
"index_patterns": ["crashplan*"],
"settings": {
"refresh_interval": "30s",
"codec": "best_compression",
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"_source": {
"enabled": true
},
"properties": {
"eventId": {
"type": "keyword"
},
"eventType": {
"type": "keyword"
},
"eventTimestamp": {
"type": "date"
},
"insertionTimestamp": {
"type": "date"
},
"filePath": {
"type": "keyword"
},
"fileName": {
"type": "keyword"
},
"fileType": {
"type": "keyword"
},
"fileCategory": {
"type": "keyword"
},
"fileSize": {
"type": "long"
},
"fileOwner": {
"type": "keyword"
},
"md5Checksum": {
"type": "keyword"
},
"sha256Checksum": {
"type": "keyword"
},
"createdTimestamp": {
"type": "date"
},
"modifyTimestamp": {
"type": "date"
},
"deviceUsername": {
"type": "keyword"
},
"username": {
"type": "alias",
"path": "deviceUsername"
},
"user": {
"type": "alias",
"path": "deviceUsername"
},
"deviceUid": {
"type": "keyword"
},
"userUid": {
"type": "keyword"
},
"osHostname": {
"type": "keyword"
},
"hostname": {
"type": "alias",
"path": "osHostname"
},
"host": {
"type": "alias",
"path": "osHostname"
},
"domainName": {
"type": "keyword"
},
"publicIpAddress": {
"type": "keyword"
},
"privateIpAddresses": {
"type": "keyword"
},
"privateIpAddress": {
"type": "alias",
"path": "privateIpAddresses"
},
"actor": {
"type": "keyword"
},
"directoryId": {
"type": "keyword"
},
"source": {
"type": "keyword"
},
"url": {
"type": "keyword"
},
"shared": {
"type": "keyword"
},
"sharedWith": {
"type": "keyword"
},
"sharingTypeAdded": {
"type": "keyword"
},
"cloudDriveId": {
"type": "keyword"
},
"detectionSourceAlias": {
"type": "keyword"
},
"fileId": {
"type": "keyword"
},
"exposure": {
"type": "keyword"
},
"processOwner": {
"type": "keyword"
},
"processName": {
"type": "keyword"
},
"removableMediaVendor": {
"type": "keyword"
},
"removableMediaName": {
"type": "keyword"
},
"removableMediaSerialNumber": {
"type": "keyword"
},
"removableMediaCapacity": {
"type": "long"
},
"removableMediaBusType": {
"type": "keyword"
},
"syncDestination": {
"type": "keyword"
},
"status": {
"type": "keyword"
},
"message": {
"type": "keyword"
},
"continent": {
"type": "keyword"
},
"continentCode": {
"type": "keyword"
},
"country": {
"type": "keyword"
},
"countryCode": {
"type": "keyword"
},
"region": {
"type": "keyword"
},
"regionName": {
"type": "keyword"
},
"city": {
"type": "keyword"
},
"district": {
"type": "keyword"
},
"zip": {
"type": "keyword"
},
"lat": {
"type": "float"
},
"lon": {
"type": "float"
},
"timezone": {
"type": "keyword"
},
"currency": {
"type": "keyword"
},
"isp": {
"type": "keyword"
},
"org": {
"type": "keyword"
},
"as": {
"type": "keyword"
},
"asname": {
"type": "keyword"
},
"reverse": {
"type": "keyword"
},
"mobile": {
"type": "boolean"
},
"proxy": {
"type": "boolean"
},
"query": {
"type": "keyword"
},
"geoPoint": {
"type": "geo_point"
}
}
},
"aliases": {
"security": { },
"crashplan": { }
}
}
Loading

0 comments on commit ff567e4

Please sign in to comment.