Active Scanning and Replacing (AWS) Tokens (#254)

.dockerignore

@@ -5,5 +5,19 @@
 .travis.yml
 Dockerfile
 spec
-#IDEs folders
-.idea
+
+# Ignore editor specific configs
+/.idea
+/.vscode
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+.generators
+.rakeTasks
+
+# System Files
+.DS_Store
+Thumbs.db

.gitignore

@@ -5,3 +5,19 @@ play
 .jdk-overlay
 .*env
 coverage/
+
+# Ignore editor specific configs
+/.idea
+/.vscode
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+.generators
+.rakeTasks
+
+# System Files
+.DS_Store
+Thumbs.db

db/deploy/create_scan_results_table.sql

@@ -0,0 +1,43 @@
+-- Deploy travis-logs:create_scan_results_table to pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  CREATE TABLE scan_results (
+    id bigint NOT NULL,
+    repository_id bigint NOT NULL,
+    job_id bigint NOT NULL,
+    log_id bigint NOT NULL,
+    owner_id integer NOT NULL,
+    owner_type character varying NOT NULL,
+    content jsonb NOT NULL,
+    issues_found integer NOT NULL,
+    archived boolean,
+    purged_at timestamp without time zone,
+    token character varying,
+    token_created_at timestamp without time zone,
+    created_at timestamp without time zone
+  );
+
+  CREATE SEQUENCE scan_results_id_seq
+  START WITH 1
+  INCREMENT BY 1
+  NO MINVALUE
+  NO MAXVALUE
+  CACHE 1;
+
+  ALTER SEQUENCE scan_results_id_seq OWNED BY scan_results.id;
+
+  ALTER TABLE ONLY scan_results
+  ALTER COLUMN id
+  SET DEFAULT nextval('scan_results_id_seq'::regclass);
+
+  ALTER TABLE ONLY scan_results
+  ADD CONSTRAINT scan_results_pkey PRIMARY KEY (id);
+
+  CREATE INDEX index_scan_results_on_repository_id
+  ON scan_results
+  USING btree (repository_id);
+
+COMMIT;

db/deploy/create_scan_tracker_table.sql

@@ -0,0 +1,32 @@
+-- Deploy travis-logs:create_scan_tracker_table to pg
+-- requires: logs_create_scan_status
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  CREATE TABLE scan_tracker (
+    id bigint NOT NULL,
+    log_id bigint NOT NULL,
+    scan_status character varying,
+    details jsonb,
+    created_at timestamp without time zone
+  );
+
+  CREATE SEQUENCE scan_tracker_id_seq
+  START WITH 1
+  INCREMENT BY 1
+  NO MINVALUE
+  NO MAXVALUE
+  CACHE 1;
+
+  ALTER SEQUENCE scan_tracker_id_seq OWNED BY scan_tracker.id;
+
+  ALTER TABLE ONLY scan_tracker
+  ALTER COLUMN id
+  SET DEFAULT nextval('scan_tracker_id_seq'::regclass);
+
+  ALTER TABLE ONLY scan_tracker
+  ADD CONSTRAINT scan_tracker_pkey PRIMARY KEY (id);
+
+COMMIT;

db/deploy/logs_create_scan_status.sql

@@ -0,0 +1,27 @@
+-- Deploy travis-logs:logs_create_scan_status to pg
+-- requires: partman_remove_constraint
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  ALTER TABLE logs
+  ADD COLUMN scan_status character varying,
+  ADD COLUMN scan_status_updated_at timestamp without time zone,
+  ADD COLUMN censored boolean,
+  ADD COLUMN scan_queued_at timestamp without time zone,
+  ADD COLUMN scan_started_at timestamp without time zone,
+  ADD COLUMN scan_processing_at timestamp without time zone,
+  ADD COLUMN scan_finalizing_at timestamp without time zone,
+  ADD COLUMN scan_ended_at timestamp without time zone;
+
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_order_by_newest ON public.logs USING btree (scan_status, id DESC);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_and_scan_status_updated_at ON public.logs USING btree (scan_status, scan_status_updated_at);
+  -- CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_and_scan_status_updated_at_where_running ON public.logs USING btree (scan_status, scan_status_updated_at) WHERE ((scan_status)::text = ANY ((ARRAY['started'::character varying, 'processing'::character varying, 'finalizing'::character varying])::text[]));
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_queued_at ON public.logs USING btree (scan_queued_at);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_started_at ON public.logs USING btree (scan_started_at);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_processing_at ON public.logs USING btree (scan_processing_at);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_finalizing_at ON public.logs USING btree (scan_finalizing_at);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_ended_at ON public.logs USING btree (scan_ended_at);
+
+COMMIT;

db/revert/create_scan_results_table.sql

@@ -0,0 +1,9 @@
+-- Revert travis-logs:create_scan_results_table from pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  DROP TABLE scan_results;
+
+COMMIT;

db/revert/create_scan_tracker_table.sql

@@ -0,0 +1,9 @@
+-- Revert travis-logs:create_scan_tracker_table from pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  DROP TABLE scan_tracker CASCADE;
+
+COMMIT;

db/revert/logs_create_scan_status.sql

@@ -0,0 +1,25 @@
+-- Revert travis-logs:logs_create_scan_status from pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  ALTER TABLE logs
+  DROP COLUMN scan_status,
+  DROP COLUMN scan_status_updated_at,
+  DROP COLUMN censored,
+  DROP COLUMN scan_queued_at,
+  DROP COLUMN scan_started_at,
+  DROP COLUMN scan_processing_at,
+  DROP COLUMN scan_finalizing_at,
+  DROP COLUMN scan_ended_at;
+
+  DROP INDEX index_logs_on_scan_status_order_by_newest;
+  DROP INDEX index_logs_on_scan_status_and_scan_status_updated_at;
+  DROP INDEX index_logs_on_scan_queued_at;
+  DROP INDEX index_logs_on_scan_started_at;
+  DROP INDEX index_logs_on_scan_processing_at;
+  DROP INDEX index_logs_on_scan_finalizing_at;
+  DROP INDEX index_logs_on_scan_ended_at;
+
+COMMIT;

db/sqitch.plan

@@ -6,3 +6,6 @@ vacuum_settings [structure] 2017-04-04T19:37:24Z Dan Buch <[email protected]> #
 log_parts_created_at_not_null [structure] 2017-04-04T19:52:23Z Dan Buch <[email protected]> # Modify log_parts.created_at to be NOT NULL with default for use with partman
 partman [log_parts_created_at_not_null] 2017-04-04T20:24:49Z Dan Buch <[email protected]> # Enable and configure partman for log_parts
 partman_remove_constraint 2018-04-27T11:41:39Z Igor Wiedler <[email protected]> # Remove partman constraint exclusion on log_id column
+logs_create_scan_status 2022-08-05T12:21:22Z Andrii Mysko <[email protected]> # Add scan status columns to logs table
+create_scan_tracker_table 2022-08-05T12:21:23Z Andrii Mysko <[email protected]> # Add scan_tracker table
+create_scan_results_table 2022-09-05T14:31:43Z Stanislav Colotinschi <[email protected]> # Add scan_results table

db/verify/create_scan_results_table.sql

@@ -0,0 +1,11 @@
+-- Verify travis-logs:create_scan_results_table on pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  SELECT id
+  FROM scan_results
+  WHERE false;
+
+ROLLBACK;

db/verify/create_scan_tracker_table.sql

@@ -0,0 +1,11 @@
+-- Verify travis-logs:create_scan_tracker_table on pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  SELECT id, scan_status, details, created_at
+  FROM scan_tracker
+  WHERE false;
+
+COMMIT;

db/verify/logs_create_scan_status.sql

@@ -0,0 +1,11 @@
+-- Verify travis-logs:logs_create_scan_status on pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  SELECT scan_status, scan_status_updated_at, censored, scan_queued_at, scan_started_at, scan_processing_at, scan_finalizing_at, scan_ended_at
+  FROM logs
+  WHERE false;
+
+COMMIT;

lib/travis/logs/database.rb

@@ -149,6 +149,24 @@ def create_log(job_id)
         db[:logs].insert(job_id: job_id, created_at: now, updated_at: now)
       end
 
+      def create_scan_tracker_entry(log_id, scan_status)
+        maint.restrict!
+        db[:scan_tracker].insert({
+          log_id: log_id,
+          scan_status: scan_status,
+          created_at: Time.now.utc
+        })
+      end
+
+      def update_log_scan_status(log_id, scan_status)
+        db.transaction do
+          db[:logs]
+            .where(id: log_id)
+            .update(scan_status_updated_at: Time.now.utc, scan_status: scan_status)
+          create_scan_tracker_entry(log_id, scan_status)
+        end
+      end
+
       def create_log_part(params)
         maint.restrict!
         db[:log_parts].insert(params.merge(created_at: Time.now.utc))

lib/travis/logs/s3.rb

@@ -10,7 +10,7 @@ module Logs
     class S3
       def self.setup
         Aws.config.update(
-          region: 'us-east-1',
+          region: ENV['TRAVIS_LOGS_S3_REGION'] || 'us-east-1',
           credentials: Aws::Credentials.new(
             Travis.config.s3.access_key_id,
             Travis.config.s3.secret_access_key

lib/travis/logs/services/aggregate_logs.rb

@@ -96,6 +96,7 @@ def aggregate_log(log_id)
           measure do
             database.db.transaction do
               aggregate(log_id)
+              database.update_log_scan_status(log_id, 'ready_for_scan')
               clean(log_id) unless skip_empty? && log_empty?(log_id)
             end
           end