Add support for Packer

.gitignore

@@ -3,3 +3,7 @@
 *.log
 *.ova
 *.retry
+
+.project
+manifest.json
+**/*.box

build_packer.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -eux
+
+# set the session to be noninteractive
+export DEBIAN_FRONTEND="noninteractive"
+
+### set region
+export AWS_REGION="us-east-1"
+
+### list first VPC id
+export BUILD_VPC_ID=$(aws ec2 describe-vpcs \
+	--query 'Vpcs[0].[VpcId]' \
+	--output text);
+echo $BUILD_VPC_ID;
+
+### list first subnet id, within VPC
+export BUILD_SUBNET_ID=$(aws ec2 describe-subnets \
+	--filters "Name=vpc-id,Values=$BUILD_VPC_ID" \
+	--query 'Subnets[0].[SubnetId]' \
+	--output text);
+echo $BUILD_SUBNET_ID;
+
+### build Packer AMI
+
+packer validate packer.json
+
+packer inspect packer.json
+
+packer build packer.json
+
+export BUILD_AMI_ID=$(jq '.builds[-1].artifact_id' -r manifest.json | cut -d':' -f2);
+echo $BUILD_AMI_ID;

default_configs/elasticsearch.yml

@@ -0,0 +1,88 @@
+# ======================== Elasticsearch Configuration =========================
+#
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+#       Before you set out to tweak and tune the configuration, make sure you
+#       understand what are you trying to accomplish and the consequences.
+#
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
+#
+# Please consult the documentation for further information on configuration options:
+# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
+#
+# ---------------------------------- Cluster -----------------------------------
+#
+# Use a descriptive name for your cluster:
+#
+#cluster.name: my-application
+#
+# ------------------------------------ Node ------------------------------------
+#
+# Use a descriptive name for the node:
+#
+#node.name: node-1
+#
+# Add custom attributes to the node:
+#
+#node.attr.rack: r1
+#
+# ----------------------------------- Paths ------------------------------------
+#
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+#path.data: /path/to/data
+#
+# Path to log files:
+#
+#path.logs: /path/to/logs
+#
+# ----------------------------------- Memory -----------------------------------
+#
+# Lock the memory on startup:
+#
+bootstrap.memory_lock: true
+#
+# Make sure that the heap size is set to about half the memory available
+# on the system and that the owner of the process is allowed to use this
+# limit.
+#
+# Elasticsearch performs poorly when the system is swapping the memory.
+#
+# ---------------------------------- Network -----------------------------------
+#
+# Set the bind address to a specific IP (IPv4 or IPv6):
+#
+#network.host: 192.168.0.1
+#
+# Set a custom port for HTTP:
+#
+#http.port: 9200
+#
+# For more information, consult the network module documentation.
+#
+# --------------------------------- Discovery ----------------------------------
+#
+# Pass an initial list of hosts to perform discovery when new node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
+#
+#discovery.zen.ping.unicast.hosts: ["host1", "host2"]
+#
+# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1):
+#
+discovery.zen.minimum_master_nodes: 1
+#
+# For more information, consult the zen discovery module documentation.
+#
+# ---------------------------------- Gateway -----------------------------------
+#
+# Block initial recovery after a full cluster restart until N nodes are started:
+#
+#gateway.recover_after_nodes: 3
+#
+# For more information, consult the gateway module documentation.
+#
+# ---------------------------------- Various -----------------------------------
+#
+# Require explicit names when deleting indices:
+#
+action.destructive_requires_name: true

default_configs/filebeat.yml

@@ -0,0 +1,33 @@
+filebeat.prospectors:
+
+#- input_type: log
+#  paths:
+#    - /var/log/syslog
+#  document_type: syslog
+
+- input_type: log
+  paths:
+    - /var/log/kibana/kibana.log
+  document_type: kibana
+  json.message_key: log
+
+- input_type: log
+  paths:
+    - /var/lib/docker/containers/*/*.log
+  document_type: docker
+  json.message_key: log
+
+
+filebeat.modules:
+- module: system
+- module: nginx
+
+
+#output.logstash:
+#  hosts: ["localhost:5044"]
+
+
+output.elasticsearch:
+  hosts: ["localhost:9200"]
+  username: "elastic"
+  password: "changeme"

default_configs/heartbeat.yml

@@ -0,0 +1,11 @@
+heartbeat.monitors:
+- type: http
+  urls: ["http://localhost:80"]
+  schedule: '@every 10s'
+  check.response.status: 200
+
+
+output.elasticsearch:
+  hosts: ["localhost:9200"]
+  username: "elastic"
+  password: "changeme"

default_configs/kibana.yml

@@ -0,0 +1,106 @@
+# Kibana is served by a back end server. This setting specifies the port to use.
+#server.port: 5601
+
+# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
+# The default is 'localhost', which usually means remote machines will not be able to connect.
+# To allow connections from remote users, set this parameter to a non-loopback address.
+#server.host: "localhost"
+
+# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects
+# the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests
+# to Kibana. This setting cannot end in a slash.
+#server.basePath: ""
+
+# The maximum payload size in bytes for incoming server requests.
+#server.maxPayloadBytes: 1048576
+
+# The Kibana server's name.  This is used for display purposes.
+#server.name: "your-hostname"
+
+# The URL of the Elasticsearch instance to use for all your queries.
+#elasticsearch.url: "http://localhost:9200"
+
+# When this setting's value is true Kibana uses the hostname specified in the server.host
+# setting. When the value of this setting is false, Kibana uses the hostname of the host
+# that connects to this Kibana instance.
+#elasticsearch.preserveHost: true
+
+# Kibana uses an index in Elasticsearch to store saved searches, visualizations and
+# dashboards. Kibana creates a new index if the index doesn't already exist.
+#kibana.index: ".kibana"
+
+# The default application to load.
+#kibana.defaultAppId: "discover"
+
+# If your Elasticsearch is protected with basic authentication, these settings provide
+# the username and password that the Kibana server uses to perform maintenance on the Kibana
+# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
+# is proxied through the Kibana server.
+#elasticsearch.username: "user"
+#elasticsearch.password: "pass"
+
+# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively.
+# These settings enable SSL for outgoing requests from the Kibana server to the browser.
+#server.ssl.enabled: false
+#server.ssl.certificate: /path/to/your/server.crt
+#server.ssl.key: /path/to/your/server.key
+
+# Optional settings that provide the paths to the PEM-format SSL certificate and key files.
+# These files validate that your Elasticsearch backend uses the same key files.
+#elasticsearch.ssl.certificate: /path/to/your/client.crt
+#elasticsearch.ssl.key: /path/to/your/client.key
+
+# Optional setting that enables you to specify a path to the PEM file for the certificate
+# authority for your Elasticsearch instance.
+#elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ]
+
+# To disregard the validity of SSL certificates, change this setting's value to 'none'.
+#elasticsearch.ssl.verificationMode: full
+
+# Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
+# the elasticsearch.requestTimeout setting.
+#elasticsearch.pingTimeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
+# must be a positive integer.
+#elasticsearch.requestTimeout: 30000
+
+# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
+# headers, set this value to [] (an empty list).
+#elasticsearch.requestHeadersWhitelist: [ authorization ]
+
+# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
+# by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
+#elasticsearch.customHeaders: {}
+
+# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
+#elasticsearch.shardTimeout: 0
+
+# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
+#elasticsearch.startupTimeout: 5000
+
+# Specifies the path where Kibana creates the process ID file.
+#pid.file: /var/run/kibana.pid
+
+# Enables you specify a file where Kibana stores log output.
+#logging.dest: stdout
+
+# Set the value of this setting to true to suppress all logging output.
+#logging.silent: false
+
+# Set the value of this setting to true to suppress all logging output other than error messages.
+#logging.quiet: false
+
+# Set the value of this setting to true to log all events, including system usage information
+# and all requests.
+#logging.verbose: false
+
+# Set the interval in milliseconds to sample system and process performance
+# metrics. Minimum is 100ms. Defaults to 5000.
+#ops.interval: 5000
+
+# The default locale. This locale can be used in certain circumstances to substitute any missing
+# translations.
+#i18n.defaultLocale: "en"
+server.host: "0.0.0.0"
+logging.dest: /var/log/kibana/kibana.log

default_configs/metricbeat.yml

@@ -0,0 +1,47 @@
+metricbeat.modules:
+
+
+- module: system
+  metricsets:
+    - cpu
+    - load
+    - core
+    - diskio
+    - filesystem
+    - fsstat
+    - memory
+    - network
+    - process
+    - socket
+  enabled: true
+  period: 10s
+  processes: ['.*']
+  cgroups: true
+
+- module: redis
+  metricsets: ["info"]
+  enabled: true
+  period: 10s
+  hosts: ["127.0.0.1:6379"]
+
+- module: nginx
+  metricsets: ["stubstatus"]
+  enabled: true
+  period: 10s
+  hosts: ["http://127.0.0.1"]
+
+- module: mongodb
+  metricsets: ["status"]
+  hosts: ["localhost:27017"]
+
+- module: docker
+  metricsets: ["cpu", "info", "memory", "network", "diskio", "container"]
+  hosts: ["unix:///var/run/docker.sock"]
+  enabled: true
+  period: 10s
+
+
+output.elasticsearch:
+  hosts: ["http://localhost:9200"]
+  username: "elastic"
+  password: "changeme"

default_configs/packetbeat.yml

@@ -0,0 +1,35 @@
+packetbeat.interfaces.device: any
+
+
+packetbeat.flows:
+  timeout: 30s
+  period: 10s
+
+
+packetbeat.protocols.icmp:
+  enabled: true
+
+
+packetbeat.protocols.dns:
+  ports: [53]
+  include_authorities: true
+  include_additionals: true
+
+
+packetbeat.protocols.http:
+  ports: [80, 5601]
+
+
+packetbeat.protocols.redis:
+  ports: [6379]
+
+
+packetbeat.protocols.mongodb:
+  ports: [27017]
+
+
+output.redis:
+  hosts: ["localhost:6379"]
+  datatype: "list"
+  key: "packetbeat"
+  db: 0

elastic-stack/0_install.yml

@@ -46,21 +46,23 @@
     apt: name=python-pip state=latest
 
 
+  # https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-using-the-repository
   - name: Get the Docker key on 64bit
     apt_key:
-      keyserver: "hkp://p80.pool.sks-keyservers.net:80"
-      id: 58118E89F3A912897C070ADBF76221572C52609D
+      url: "https://download.docker.com/linux/ubuntu/gpg"
+      id: 0EBFCD88
       state: present
     when: "platform == 'amd64'"
 
   - name: Add the Docker repository
     apt_repository:
-      repo: "deb https://apt.dockerproject.org/repo ubuntu-xenial main"
+      repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable"
       state: present
     when: "platform == 'amd64'"
 
+  # https://docs.docker.com/release-notes/docker-ce/
   - name: Install Docker
-    apt: name=docker-engine state=latest update_cache=yes
+    apt: name=docker-ce state=latest update_cache=yes
     when: "platform == 'amd64'"
 
   - name: Make sure Docker autostarts and is started