Add cleanup for alternate needle versions

There is now a minion task which will clean up alternate needle files
created by the WebUI. The minimum retention time of the needle files can
be set in the config, and defaults to 30 minutes

lib/OpenQA/Setup.pm

@@ -64,6 +64,7 @@ sub read_config ($app) {
             do_push => 'no',
             do_cleanup => 'no',
             checkout_needles_sha => 'no',
+            minimum_needle_retention_time => undef,
         },
         'scheduler' => {
             max_job_scheduled_time => 7,

lib/OpenQA/Shared/Plugin/Gru.pm

@@ -27,7 +27,7 @@ sub register_tasks ($self) {
       for (
         qw(OpenQA::Task::AuditEvents::Limit),
         qw(OpenQA::Task::Asset::Download OpenQA::Task::Asset::Limit),
-        qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete),
+        qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete OpenQA::Task::Needle::RemoveVersions),
         qw(OpenQA::Task::Job::Limit),
         qw(OpenQA::Task::Job::ArchiveResults),
         qw(OpenQA::Task::Job::FinalizeResults),

lib/OpenQA/Task/Needle/RemoveVersions.pm

@@ -0,0 +1,38 @@
+# Copyright 2019 SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+package OpenQA::Task::Needle::RemoveVersions;
+use Mojo::Base 'Mojolicious::Plugin', -signatures;
+
+use File::Find;
+use File::stat;
+
+my $minimum_retention_time;
+
+sub register ($self, $app, $job) {
+    $minimum_retention_time = $app->config->{'scm git'}->{minimum_needle_retention_time} * 60 // 30 * 60;
+    $app->minion->add_task(remove_needle_versions => sub ($job) { _remove_needle_versions($app, $job) });
+}
+
+sub _remove_needle_versions ($app, $job) {
+    return $job->finish({error => 'Another job to remove needle versions is running. Try again later.'})
+      unless my $guard = $app->minion->guard('limit_needle_versions_task', 7200);
+
+    my $needle_versions_path = "/tmp/needle_dirs";
+    return unless -d $needle_versions_path;
+    # Remove all temporary needles which haven't been accessed in time period specified in config
+    find(\&wanted, $needle_versions_path);
+    return;
+}
+
+sub wanted() {
+    my $filepath = $File::Find::name;
+    return unless -f $filepath;
+    my $now = time;
+    my $atime = stat($filepath)->atime;
+    if (($now - $atime) > $minimum_retention_time) {
+        unlink($filepath);
+    }
+}
+
+1;

lib/OpenQA/Utils.pm

@@ -256,7 +256,7 @@ sub locate_needle {
 
     if ($needles_ref) {
         my $needles_dir_basename = basename(dirname($needles_dir));
-        my $temp_needles_dir = "/tmp/$needles_dir_basename/worktrees/$needles_ref/needles";
+        my $temp_needles_dir = "/tmp/needle_dirs/$needles_dir_basename/$needles_ref/needles";
         if (File::Spec->splitdir($relative_needle_path) > 1) {
             make_path($temp_needles_dir . '/' . dirname($relative_needle_path));
         }

lib/OpenQA/WebAPI/Controller/Step.pm

@@ -110,7 +110,7 @@ sub _create_tmpdir_for_needles_refspec ($self, $job) {
     chomp($needles_ref);
     return undef unless $needles_ref;
     my $needle_dir_basename = basename(dirname($needle_dir));
-    my $new_path = "/tmp/$needle_dir_basename/worktrees/$needles_ref/needles";
+    my $new_path = "/tmp/needle_dirs/$needle_dir_basename/$needles_ref/needles";
     if (!-d $new_path) {
         make_path($new_path);
         qx{git -C "$needle_dir" fetch --depth 1 origin "$needles_ref" &>/dev/null};

script/openqa-enqueue-needle-versions-cleanup

@@ -0,0 +1,2 @@
+#!/bin/sh -e
+exec "$(dirname "$0")"/openqa eval -m production -V 'app->gru->enqueue(remove_needle_versions => [], {priority => 5, ttl => 172800, limit => 1})' "$@"