Update wrappers for latest pdnssoc-cli

CERN-CERT · Oct 13, 2023 · ca7f4ef · ca7f4ef
1 parent fb8e070
commit ca7f4ef
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 61 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -50,5 +50,6 @@ jobs:
  ghcr.io/cern-cert/pdnssoc:${VERSION}
  labels: ${{ steps.meta.outputs.labels }}
  build-args: |
- GO_DNSCOLLECTOR_VERSION=0.35.0
- PDNSSOC_CLI_VERSION=0.0.2.dev5
+ GO_DNSCOLLECTOR_VERSION=0.36.0
+ PDNSSOC_CLI_VERSION=0.0.2
+ PDNSSOC_CLI_PYPI_INDEX=pypi.org
diff --git a/.github/workflows/release_rolling.yml b/.github/workflows/release_rolling.yml
@@ -42,5 +42,6 @@ jobs:
  ghcr.io/cern-cert/pdnssoc:edge
  labels: ${{ steps.meta.outputs.labels }}
  build-args: |
- GO_DNSCOLLECTOR_VERSION=0.35.0
- PDNSSOC_CLI_VERSION=0.0.2.dev10
+ GO_DNSCOLLECTOR_VERSION=0.36.0
+ PDNSSOC_CLI_VERSION=0.0.2.dev26
+ PDNSSOC_CLI_PYPI_INDEX=test.pypi.org
diff --git a/files/configuration/pdnssoccli/notification_email.html b/files/configuration/pdnssoccli/notification_email.html
@@ -32,49 +32,59 @@
  </style>
  </head>
  <body>
+ {% for sensor, alert_data in alerts.items() %}
+ <h1>{{ sensor }}</h1>
  <table>
  <tbody>
  <tr>
- <th>pDNSSOC client</th>
- <th>First Occurrence</th>
- <th>IoCs detected</th>
- <th>MISP event</th>
- <th>Total # of IoCs</th>
+ <th>DNS client</th>
+ <th>Query</th>
+ <th>Answers</th>
+ <th>First DNS query</th>
+ <th>MISP IOC</th>
+ <th>MISP Event</th>
  <th>Publication</th>
- <th>Organisation</th>
+ <th>Organization</th>
  <th>Comment</th>
  <th>Tags</th>
  </tr>
- <% all_results.each do |ioc, data_ioc| %>
- <tr>
- <% if data_ioc["client_name"] == "" %>
- <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= data_ioc["client_ip"] %></td>
- <% else %>
- <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= data_ioc["client_name"] %> (<%= data_ioc["client_ip"] %>)</td>
- <% end %>
- <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><%= Time.at(data_ioc["first_occurrence"]).strftime(TIME_FORMAT_YMD) %></td>
- <td style="text-align: left;" rowspan="<%= data_ioc["misp"].length %>"><a href="" target="_new"><%= data_ioc["ioc_detected"] %></a></td>
- <% data_ioc["misp"].each_with_index do |misp_event, idx_event| %>
- <% if idx_event > 0 %>
- <tr>
- <% end %>
- <td style="text-align: left;"><a href="https://<%= misp_event["misp_server"] %>/events/view/<%= misp_event["misp_id"] %>" target="_new"><%= misp_event["misp_info"] %></a></td>
- <td style="text-align: left;"><%= misp_event["num_iocs"] %></td>
- <td style="text-align: left;"><%= misp_event["publication"] %></td>
- <td style="text-align: left;"><%= misp_event["organisation"] %></td>
- <td style="text-align: left;"><%= misp_event["comment"] %></td>
+
+ {% for client, client_data in alert_data.items()|sort(attribute='0') %}
+ {% for query, query_data in client_data.items()|sort(attribute='1.first_occurence') %}
+ <tr>
+ <td rowspan="{{ (query_data['events'] | length ) + 1 }}" style="text-align: left;">{{ client }}</td>
+ <td rowspan="{{ (query_data['events'] | length ) + 1 }}" style="text-align: left;">{{ query }}</td>
+
+ <td rowspan="{{ (query_data['events'] | length ) + 1 }}" style="text-align: left;">
+ {% for answer in query_data['answers'] %}
+ {{ answer }}<br>
+ {% endfor %}
+ </td>
+ <td rowspan="{{ (query_data['events'] | length ) + 1 }}" style="text-align: left;">{{ query_data['first_occurence'] }}</td>
+ {% for event_uuid, event in query_data['events'].items()|sort(attribute='1.publication')|reverse %}
+ <tr>
+ <td style="text-align: left;">{{ event["ioc"] }}</td>
+ <td style="text-align: left;"><a href="{{ event['event_url'] }}" target="_new">{{ event["info"] }}</a></td>
+ <td style="text-align: left;">{{ event["publication"] }}</td>
+ <td style="text-align: left;">{{ event["organization"] }}</td>
+ <td style="text-align: left;">{{ event["comment"] }}</td>
  <td style="text-align: left;">
- <% for tag in misp_event["tags"] do %>
- <span style="background: <%= tag["colour"] %>;">
- <b><span style="color: #fff; mix-blend-mode: difference; padding: 5px;"><%= tag["name"] %></span></b>
+ {% for tag in event["tags"]%}
+ <span style="background: {{ tag["colour"] }};">
+ <b><span style="color: #fff; mix-blend-mode: difference; padding: 5px;">{{ tag["name"] }}</span></b><br>
  </span>
- <% end %>
+ {% endfor %}
  </td>
  </tr>
- <% end %>
- </tr>
- <% end %>
+ {% endfor %}
+ </tr>
+ <tr>
+ </tr>
+ {% endfor %}
+ {% endfor %}
  </tbody>
  </table>
+
+ {% endfor %}
  </body>
-</html>
+</html>
diff --git a/files/configuration/pdnssoccli/pdnssoccli.yml b/files/configuration/pdnssoccli/pdnssoccli.yml
@@ -1,23 +1,48 @@
 logging_level: "INFO"
 
 misp_servers:
- - domain: "https://misp1.myserver.org/"
- api_key: "API_KEY_1"
+ - domain: "https://example-misp-instance.com"
+ api_key: "API_KEY"
+ # misp.search() arguments
  args:
  enforce_warninglist: True
+ periods:
+ generic:
+ delta:
+ days: 30
+ tags:
+ - names:
+ - "tag_name"
+ delta: False
 
 correlation:
- input_dir: /var/dnscollector/queries
- output_dir: /var/dnscollector/alerts
- malicious_domains_file: /var/dnscollector/misp_domains.txt
- malicious_ips_file: /var/dnscollector/misp_ips.txt
- last_correlation_pointer_file: /var/dnscollector/correlation.last
- last_retro_pointer_file: /var/dnscollector/repo.last
+ input_dir: ./input_dir/ # use this if no files are defined from commmand line
+ output_dir: ./output_dir/
+ archive_dir: ./archive/ # use this as input for looking back
+ malicious_domains_file: ./misp_domains.txt
+ malicious_ips_file: ./misp_ips.txt
+ last_correlation_pointer_file: ./correlation.last
+ last_retro_pointer_file: ./retro.last
 
 schedules:
  fetch_iocs:
  interval: 10 # minutes
  correlation:
  interval: 1 # minutes
  retro:
- interval: 1440 # minutes
+ interval: 1440 # minutes
+ alerting:
+ interval: 60 # minutes
+
+alerting:
+ last_alerting_pointer_file: ./tests/alert.last
+ email:
+ from: "[email protected]"
+ subject: "[pDNSSOC] Community XYZ alert"
+ summary_to: "[email protected]"
+ server: "localhost"
+ port: 25
+ template: ./alert_email_template.html
+ mappings:
+ - client_id: client_1
+ contact: [email protected]
diff --git a/files/docker/Dockerfile b/files/docker/Dockerfile
@@ -2,6 +2,7 @@ FROM python:3.11-alpine
 
 ARG GO_DNSCOLLECTOR_VERSION
 ARG PDNSSOC_CLI_VERSION
+ARG PDNSSOC_CLI_PYPI_INDEX
 
 RUN apk add --no-cache --update \
  bash \
@@ -14,9 +15,7 @@ RUN apk add --no-cache --update \
  gzip
 
 # Install pdnssoc-cli
-#RUN pip install "pdnssoc-cli==$PDNSSOC_CLI_VERSION"
-
-RUN pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple pdnssoc-cli==$PDNSSOC_CLI_VERSION
+RUN pip install --index-url https://$PDNSSOC_CLI_PYPI_INDEX/simple --extra-index-url https://pypi.org/simple pdnssoc-cli==$PDNSSOC_CLI_VERSION
 
 RUN mkdir /build
 

diff --git a/files/docker/fluentd/fluent.conf b/files/docker/fluentd/fluent.conf
@@ -19,13 +19,13 @@
 <match pdnssoc>
  @type opensearch
  hosts https://opensearch_instance
- index_name ${tag}-%Y.%m.%d #=> e.g.) elastic.20170811
+ index_name ${tag}-%Y.%m.%d #=> e.g.) pdnssoc-20170811
  <buffer tag, time>
  @type memory
  flush_mode immediate
  timekey 3600
  </buffer>
  user %{CHANGE_ME}
  password %{CHANGE_ME}
- ssl_verify false
+ ssl_verify true
  </match>
diff --git a/files/docker/godnscollector.yml b/files/docker/godnscollector.yml
@@ -6,7 +6,7 @@ global:
  # If turned on, log some applications messages
  trace:
  # debug informations
- verbose: true
+ verbose: false
  # Set the server identity name
  # comment the following line to use the hostname
  server-identity: "dns-collector"

diff --git a/files/docker/pdnssoccli.yml b/files/docker/pdnssoccli.yml
@@ -1,22 +1,48 @@
 logging_level: "INFO"
 
 misp_servers:
- - domain: "https://your-misp-server.com"
- api_key: "API_KEY_1"
- args: {}
+ - domain: "https://example-misp-instance.com"
+ api_key: "API_KEY"
+ # misp.search() arguments
+ args:
+ enforce_warninglist: True
+ periods:
+ generic:
+ delta:
+ days: 30
+ tags:
+ - names:
+ - "tag_name"
+ delta: False
 
 correlation:
- input_dir: /var/dnscollector/queries
- output_dir: /var/dnscollector/alerts
- malicious_domains_file: /var/dnscollector/misp_domains.txt
- malicious_ips_file: /var/dnscollector/misp_ips.txt
- last_correlation_pointer_file: /var/dnscollector/correlation.last
- last_retro_pointer_file: /var/dnscollector/repo.last
+ input_dir: ./input_dir/ # use this if no files are defined from commmand line
+ output_dir: ./output_dir/
+ archive_dir: ./archive/ # use this as input for looking back
+ malicious_domains_file: ./misp_domains.txt
+ malicious_ips_file: ./misp_ips.txt
+ last_correlation_pointer_file: ./correlation.last
+ last_retro_pointer_file: ./retro.last
 
 schedules:
  fetch_iocs:
  interval: 10 # minutes
  correlation:
  interval: 1 # minutes
  retro:
- interval: 1440 # minutes
+ interval: 1440 # minutes
+ alerting:
+ interval: 60 # minutes
+
+alerting:
+ last_alerting_pointer_file: ./tests/alert.last
+ email:
+ from: "[email protected]"
+ subject: "[pDNSSOC] Community XYZ alert"
+ summary_to: "[email protected]"
+ server: "localhost"
+ port: 25
+ template: ./alert_email_template.html
+ mappings:
+ - client_id: client_1
+ contact: [email protected]