From fd44d387a904209f304f28878f04f5a69473483b Mon Sep 17 00:00:00 2001 From: Todd Bruner Date: Tue, 29 Oct 2024 15:57:06 -0600 Subject: [PATCH] updates to --- Dockerfile | 4 +-- LICENSE | 23 ++++++++++++++ README.md | 1 + bin/inbox.pl | 3 -- lib/Scot/Inbox/Config.pm | 1 + lib/Scot/Inbox/Processor.pm | 16 ++++++++++ t/msv.t | 60 +++++++++++++++++++++++++++++++++++++ t/mtest.pl | 55 ---------------------------------- t/test.log | 24 --------------- 9 files changed, 103 insertions(+), 84 deletions(-) create mode 100644 LICENSE create mode 100755 t/msv.t delete mode 100644 t/mtest.pl delete mode 100644 t/test.log diff --git a/Dockerfile b/Dockerfile index 52ffcbc..67caa82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,8 @@ RUN mkdir -p /opt/scot4-inbox && mkdir -p /var/log/scot COPY . /opt/scot4-inbox # create user/group for scotinbox -RUN groupadd scotinbox && \ - useradd -c "Scot Inbox User" -g "scotinbox" -d /opt/scot4-inbox -M -s /bin/bash scotinbox && \ +RUN groupadd -g 5555 scotinbox && \ + useradd -c "Scot Inbox User" -g "scotinbox" -u 5555 -d /opt/scot4-inbox -M -s /bin/bash scotinbox && \ chown -R scotinbox:scotinbox /opt/scot4-inbox && \ chown -R scotinbox:scotinbox /var/log/scot diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..85bb47d --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +Copyright (2024) Sandia Corporation. Under the terms of Contract DE-AC04-94AL85000, there is a non-exclusive license for use of this work by or on behalf of the U.S. Government. Export of this program may require a license from the United States Government. + +NOTICE: + +For five (5) years from 09/01/2024, the United States Government is granted for itself and others acting on its behalf a paid-up, nonexclusive, irrevocable worldwide license in this data to reproduce, prepare derivative works, and perform publicly and display publicly, by or on behalf of the Government. There is provision for the possible extension of the term of this license. Subsequent to that period or any extension granted, the United States Government is granted for itself and others acting on its behalf a paid-up, nonexclusive, irrevocable worldwide license in this data to reproduce, prepare derivative works, distribute copies to the public, perform publicly and display publicly, and to permit others to do so. The specific term of the license can be identified by inquiry made to Sandia Corporation or DOE. + +NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR SANDIA CORPORATION, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. + +Any licensee of this software has the obligation and responsibility to abide by the applicable export control laws, regulations, and general prohibitions relating to the export of technical data. Failure to obtain an export control license or other authority from the Government may result in criminal liability under U.S. laws. + +Copyright [2024] Sandia Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 1c9ac1e..a80f3e3 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,4 @@ Creates a container to run the SCOT4 inbox processor. The processor will read I This is a convenient way to input data from detection systems, threat intel feeds, and analysts directly into SCOT. For more information and instructions, consult the main SCOT4 documentaton Repository. + https://sandialabs.github.io/scot4-docs/index.html diff --git a/bin/inbox.pl b/bin/inbox.pl index 7e99e92..7f1290c 100755 --- a/bin/inbox.pl +++ b/bin/inbox.pl @@ -36,7 +36,6 @@ END EOF GetOptions( - 'config=s' => \$configfile, 'test' => \$test, 'secrets=s' => \$secrets, 'msv' => \$nomsv, @@ -47,7 +46,6 @@ END usage: $0 [--test] overwrites peeking to true - [--config=/path/to/inbox.conf] use this file as the configuration file [--secrets=/path/to/secrets.conf] use this file for secret storage [--msv] do not filter msv data [--msvlog=/path/to/log] where to log msv hits @@ -60,7 +58,6 @@ END } my $opts = { - configfile => $configfile, test => $test, secrets => $secrets, msv => $msv, diff --git a/lib/Scot/Inbox/Config.pm b/lib/Scot/Inbox/Config.pm index ffe75c6..e29cb5a 100644 --- a/lib/Scot/Inbox/Config.pm +++ b/lib/Scot/Inbox/Config.pm @@ -106,6 +106,7 @@ log4perl.appender.InboxLog.layout.ConversionPattern = %d %7p [%P] %15F{1}: %4L % scot_queue => $ENV{S4INBOX_SCOT_INPUT_QUEUE}, class => $ENV{S4INBOX_MAIL_CLIENT_CLASS}, mboxconfig => $mboxconf, + addsplunksigs => $ENV{S4INBOX_ADD_SPLUNK_SIGS}, # for disconnected nets }; return $config; } diff --git a/lib/Scot/Inbox/Processor.pm b/lib/Scot/Inbox/Processor.pm index 398a992..9aef1a8 100644 --- a/lib/Scot/Inbox/Processor.pm +++ b/lib/Scot/Inbox/Processor.pm @@ -335,6 +335,9 @@ sub create_alertgroup ($self, $json) { {filter=>\&Dumper, value=>$response}); }; $self->log->debug("response => ",{filter=>\&Dumper, value=>$rhash}); + + $self->upsert_signature($json) if (defined $self->config->{addsplunksigs}); + return 1; } $self->log->error("undefined response from ScotApi!"); @@ -342,6 +345,19 @@ sub create_alertgroup ($self, $json) { return; } +sub upsert_signature ($self, $json) { + # For a certain disconnected network, there is no dag to pull signatures into SCOT + # but the email includes the signature, so "upsert" a signature + # XXX + # query api to see if signature matches + if ($self->scotapi->signature_exists($json->{search})) { + $self->log->debug("Signature exists. moving on..."); + return; + } + + $self->scotapi->create_signature($json->{search}); +} + sub process_splunk_alert ($self, $msg) { $self->log->debug("Processing a splunk generated alert..."); diff --git a/t/msv.t b/t/msv.t new file mode 100755 index 0000000..e97a25e --- /dev/null +++ b/t/msv.t @@ -0,0 +1,60 @@ +#!/opt/perl/bin/perl + +use Test::Most; +use Data::Dumper::Concise; +use lib '../lib'; +use Scot::Inbox::Processor; +use Scot::Inbox::Config; +use Scot::Inbox::Log; +use Storable qw(dclone); +use feature qw(say); + +$ENV{S4INBOX_IMAP_SERVERNAME} = "mail.sandia.gov"; +$ENV{S4INBOX_IMAP_PORT} = 993; +$ENV{S4INBOX_IMAP_INBOX} = 'INBOX'; +$ENV{S4INBOX_IMAP_USERNAME} = 'scot-alerts'; +$ENV{S4INBOX_IMAP_PASSWORD} = ''; +$ENV{S4INBOX_SSL_VERIFY} = 1; +$ENV{S4INBOX_IMAP_PEEK} = 1; +$ENV{S4INBOX_PERMITTED_SENDERS} = '*,tbruner@sandia.gov'; +$ENV{S4INBOX_MSV_FILTER_DEFINITIONS} = '../etc/msv.defs'; +$ENV{S4INBOX_LOG_LEVEL} = 'TRACE'; +$ENV{S4INBOX_LOG_FILE} = './test.log'; +$ENV{S4INBOX_SCOTAPI_INSECURE_SSL} = 0; +$ENV{S4INBOX_API_KEY} = ''; +$ENV{S4INBOX_API_URI_ROOT} = 'https://scot4-qual/api/v1'; +$ENV{S4INBOX_MSV_DBM_FILE} = '../var/msgids.dbm'; +$ENV{S4INBOX_MAIL_CLIENT_CLASS} = 'Scot::Inbox::Imap'; +$ENV{S4INBOX_SCOT_INPUT_QUEUE} = 'alertgroup'; + +my $config = build_config(); +my $log = start_logging($config->{log}); +my $proc = Scot::Inbox::Processor->new( + config => $config, + log => $log, +); + +my $json = { + message_id => '<1234567@098765>', + data => [ + { + domain => 'ct-salsa.ca.sandia.gov', + stuff => 'goes here', + }, + { + domain => 'www.google.com', + stuff => 'dies here', + }, + ], +}; +my $json2 = dclone($json); + +$proc->filter_msv($json); + +say Dumper($json); + +$proc->filter_msv($json2); + +say Dumper($json2); + + diff --git a/t/mtest.pl b/t/mtest.pl deleted file mode 100644 index dd3b5fd..0000000 --- a/t/mtest.pl +++ /dev/null @@ -1,55 +0,0 @@ -#!/opt/perl/bin/perl -# -use feature 'signatures'; -sub is_permitted ($from, $cleared) { - print ("does $from match $cleared?\n"); - - return 1 if (regex_match($cleared, $from)); - return 1 if (wildcard_match($cleared)); - return 1 if (loose_match($cleared, $from)); - return 1 if (explicit_match($cleared, $from)); - - print ("Failed to match $from to $cleared\n\n"); - return undef; -} - -sub regex_match ( $ok, $from) { - if ( ref($ok) eq 'Regexp' ) { - print ("checking for regex match\n"); - return $from =~ /$ok/; - } - return undef; -} - -sub wildcard_match ($ok) { - print ("checking for wildcard\n"); - return $ok eq '*'; -} - -sub explicit_match ($ok, $from){ - print ("Explicit match check $ok eq $from\n"); - return $ok eq $from; -} - -sub loose_match ($ok, $from) { - print ("Loose match check $ok eq $from\n"); - return $from =~ /$ok/; -} - -my @permitted = (qw(foo@bar.com boom@bam.gov)); -my @from = ( - 'Boom, J Chuck boom@bam.gov', - 'foo@foo.com', - 'Bar, Foo foo@bar.com', - 'bing@bada.com', -); - -foreach my $f (@from) { - print "FROM: $f\n"; - foreach my $p (@permitted) { - print "P = $p\n"; - if (is_permitted($f, $p)) { - print "$_ is permitted\n\n"; - } - } -} diff --git a/t/test.log b/t/test.log deleted file mode 100644 index 05a0249..0000000 --- a/t/test.log +++ /dev/null @@ -1,24 +0,0 @@ -2023/10/09 15:36:15 INFO [347822] Log.pm: 34 ./msv.t logging to Inbox... -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 136 Checking for MSV data -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 181 MSV Scan row: { - domain => "ct-salsa.ca.sandia.gov", - stuff => "goes here", -} -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 190 Found msv item ct-salsa in row -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 181 MSV Scan row: { - domain => "www.google.com", - stuff => "dies here", -} -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 172 Found MSV items in data -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 208 wrote row to ../var/log/msv.log -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 136 Checking for MSV data -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 181 MSV Scan row: { - domain => "ct-salsa.ca.sandia.gov", - stuff => "goes here", -} -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 190 Found msv item ct-salsa in row -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 181 MSV Scan row: { - domain => "www.google.com", - stuff => "dies here", -} -2023/10/09 15:36:15 DEBUG [347822] Processor.pm: 165 Seen <1234567@098765> already, not logging