From bbe3787cd2122881f9360a7d40f1066f31003b99 Mon Sep 17 00:00:00 2001
From: Scott Clarke <scott.clarke@codethink.co.uk>
Date: Thu, 8 Jun 2023 16:25:18 +0100
Subject: [PATCH] Add cleanup for alternate needle versions

There is now a minion task which will clean up alternate needle files
created by the WebUI. The minimum retention time of the needle files can
be set in the config, and defaults to 30 minutes
---
 lib/OpenQA/Setup.pm                           |  1 +
 lib/OpenQA/Shared/Plugin/Gru.pm               |  2 +-
 lib/OpenQA/Task/Needle/RemoveVersions.pm      | 38 +++++++++++++++++++
 lib/OpenQA/Utils.pm                           |  2 +-
 lib/OpenQA/WebAPI/Controller/Step.pm          |  2 +-
 script/openqa-enqueue-needle-versions-cleanup |  2 +
 6 files changed, 44 insertions(+), 3 deletions(-)
 create mode 100644 lib/OpenQA/Task/Needle/RemoveVersions.pm
 create mode 100755 script/openqa-enqueue-needle-versions-cleanup

diff --git a/lib/OpenQA/Setup.pm b/lib/OpenQA/Setup.pm
index 7be85abe2d04..334b323c555b 100644
--- a/lib/OpenQA/Setup.pm
+++ b/lib/OpenQA/Setup.pm
@@ -64,6 +64,7 @@ sub read_config ($app) {
             do_push => 'no',
             do_cleanup => 'no',
             checkout_needles_sha => 'no',
+            minimum_needle_retention_time => undef,
         },
         'scheduler' => {
             max_job_scheduled_time => 7,
diff --git a/lib/OpenQA/Shared/Plugin/Gru.pm b/lib/OpenQA/Shared/Plugin/Gru.pm
index 7a451b184643..6bea7d9657ea 100644
--- a/lib/OpenQA/Shared/Plugin/Gru.pm
+++ b/lib/OpenQA/Shared/Plugin/Gru.pm
@@ -27,7 +27,7 @@ sub register_tasks ($self) {
       for (
         qw(OpenQA::Task::AuditEvents::Limit),
         qw(OpenQA::Task::Asset::Download OpenQA::Task::Asset::Limit),
-        qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete),
+        qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete OpenQA::Task::Needle::RemoveVersions),
         qw(OpenQA::Task::Job::Limit),
         qw(OpenQA::Task::Job::ArchiveResults),
         qw(OpenQA::Task::Job::FinalizeResults),
diff --git a/lib/OpenQA/Task/Needle/RemoveVersions.pm b/lib/OpenQA/Task/Needle/RemoveVersions.pm
new file mode 100644
index 000000000000..377220683710
--- /dev/null
+++ b/lib/OpenQA/Task/Needle/RemoveVersions.pm
@@ -0,0 +1,38 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+package OpenQA::Task::Needle::RemoveVersions;
+use Mojo::Base 'Mojolicious::Plugin', -signatures;
+
+use File::Find;
+use File::stat;
+
+my $minimum_retention_time;
+
+sub register ($self, $app, $job) {
+    $minimum_retention_time = $app->config->{'scm git'}->{minimum_needle_retention_time} * 60 // 30 * 60;
+    $app->minion->add_task(remove_needle_versions => sub ($job) { _remove_needle_versions($app, $job) });
+}
+
+sub _remove_needle_versions ($app, $job) {
+    return $job->finish({error => 'Another job to remove needle versions is running. Try again later.'})
+      unless my $guard = $app->minion->guard('limit_needle_versions_task', 7200);
+
+    my $needle_versions_path = "/tmp/needle_dirs";
+    return unless -d $needle_versions_path;
+    # Remove all temporary needles which haven't been accessed in time period specified in config
+    find(\&wanted, $needle_versions_path);
+    return;
+}
+
+sub wanted() {
+    my $filepath = $File::Find::name;
+    return unless -f $filepath;
+    my $now = time;
+    my $atime = stat($filepath)->atime;
+    if (($now - $atime) > $minimum_retention_time) {
+        unlink($filepath);
+    }
+}
+
+1;
diff --git a/lib/OpenQA/Utils.pm b/lib/OpenQA/Utils.pm
index 384f0c3eb703..39ddf28da1c4 100644
--- a/lib/OpenQA/Utils.pm
+++ b/lib/OpenQA/Utils.pm
@@ -256,7 +256,7 @@ sub locate_needle {
 
     if ($needles_ref) {
         my $needles_dir_basename = basename(dirname($needles_dir));
-        my $temp_needles_dir = "/tmp/$needles_dir_basename/worktrees/$needles_ref/needles";
+        my $temp_needles_dir = "/tmp/needle_dirs/$needles_dir_basename/$needles_ref/needles";
         if (File::Spec->splitdir($relative_needle_path) > 1) {
             make_path($temp_needles_dir . '/' . dirname($relative_needle_path));
         }
diff --git a/lib/OpenQA/WebAPI/Controller/Step.pm b/lib/OpenQA/WebAPI/Controller/Step.pm
index 2da5c69315e9..3abee1baca0a 100644
--- a/lib/OpenQA/WebAPI/Controller/Step.pm
+++ b/lib/OpenQA/WebAPI/Controller/Step.pm
@@ -110,7 +110,7 @@ sub _create_tmpdir_for_needles_refspec ($self, $job) {
     chomp($needles_ref);
     return undef unless $needles_ref;
     my $needle_dir_basename = basename(dirname($needle_dir));
-    my $new_path = "/tmp/$needle_dir_basename/worktrees/$needles_ref/needles";
+    my $new_path = "/tmp/needle_dirs/$needle_dir_basename/$needles_ref/needles";
     if (!-d $new_path) {
         make_path($new_path);
         qx{git -C "$needle_dir" fetch --depth 1 origin "$needles_ref" &>/dev/null};
diff --git a/script/openqa-enqueue-needle-versions-cleanup b/script/openqa-enqueue-needle-versions-cleanup
new file mode 100755
index 000000000000..84f71de4a245
--- /dev/null
+++ b/script/openqa-enqueue-needle-versions-cleanup
@@ -0,0 +1,2 @@
+#!/bin/sh -e
+exec "$(dirname "$0")"/openqa eval -m production -V 'app->gru->enqueue(remove_needle_versions => [], {priority => 5, ttl => 172800, limit => 1})' "$@"