Use needles from correct ref of NEEDLES_DIR

etc/openqa/openqa.ini

@@ -117,6 +117,10 @@
 #git_auto_clone = yes
 ## Experimental - Ensure a git update of all test code and needles
 #git_auto_update = no
+# whether openQA should attempt to display needles of the correct version in the web UI
+#checkout_needles_sha = no
+# retention for storing temporary needle refs in minutes
+#temp_needle_refs_retention = 120
 
 ## Authentication method to use for user management
 [auth]

lib/OpenQA/Git.pm

@@ -6,6 +6,7 @@
 use Mojo::Base -base, -signatures;
 use Mojo::Util 'trim';
 use Cwd 'abs_path';
+use Mojo::File 'path';
 use OpenQA::Utils qw(run_cmd_with_log_return_error);
 
 has 'app';
@@ -145,4 +146,16 @@
     return $r->{status};
 }
 
+sub cache_ref ($self, $ref, $relative_path, $output_file) {
+    if (-f $output_file) {
+        eval { path($output_file)->touch };
+        return $@ ? $@ : undef;
+    }
+    my @git = $self->_prepare_git_command;
+    my $res = run_cmd_with_log_return_error [@git, 'show', "$ref:./$relative_path"], output_file => $output_file;
+    return undef if $res->{status};
+    unlink $output_file;
+    return _format_git_error($res, 'Unable to cache Git ref');
+}
+
 1;

lib/OpenQA/Needles.pm

@@ -0,0 +1,57 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+package OpenQA::Needles;
+use Mojo::Base -strict, -signatures;
+
+use Exporter qw(import);
+use File::Basename;
+use File::Spec;
+use File::Spec::Functions qw(catdir);
+use OpenQA::Git;
+use OpenQA::Log qw(log_error);
+use OpenQA::Utils qw(prjdir sharedir);
+use Mojo::File qw(path);
+
+our @EXPORT = qw(temp_dir is_in_temp_dir needle_temp_dir locate_needle);
+
+my $tmp_dir = prjdir() . '/webui/cache/needle-refs';
+
+sub temp_dir () { $tmp_dir }
+
+sub is_in_temp_dir ($file_path) { index($file_path, $tmp_dir) == 0 }
+
+sub needle_temp_dir ($dir, $ref) { path($tmp_dir, basename(dirname($dir)), $ref, 'needles') }
+
+sub _locate_needle_for_ref ($relative_needle_path, $needles_dir, $needles_ref) {
+    return undef unless defined $needles_ref;
+
+    my $temp_needles_dir = needle_temp_dir($needles_dir, $needles_ref);
+    my $subdir = dirname($relative_needle_path);
+    path($temp_needles_dir, $subdir)->make_path if File::Spec->splitdir($relative_needle_path) > 1;
+
+    my $git = OpenQA::Git->new(dir => $needles_dir);
+    my $temp_json_path = "$temp_needles_dir/$relative_needle_path";
+    my $basename = basename($relative_needle_path, '.json');
+    my $relative_png_path = "$subdir/$basename.png";
+    my $temp_png_path = "$temp_needles_dir/$relative_png_path";
+    my $error = $git->cache_ref($needles_ref, $relative_needle_path, $temp_json_path)
+      // $git->cache_ref($needles_ref, $relative_png_path, $temp_png_path);
+    return $temp_json_path unless defined $error;
+    log_error "An error occurred when looking for ref '$needles_ref' of '$relative_needle_path': $error";
+    return undef;
+}
+
+sub locate_needle ($relative_needle_path, $needles_dir, $needles_ref = undef) {
+    my $location_for_ref = _locate_needle_for_ref($relative_needle_path, $needles_dir, $needles_ref);
+    return $location_for_ref if defined $location_for_ref;
+    my $absolute_filename = catdir($needles_dir, $relative_needle_path);
+    my $needle_exists = -f $absolute_filename;
+    if (!$needle_exists) {
+        $absolute_filename = catdir(sharedir(), $relative_needle_path);
+        $needle_exists = -f $absolute_filename;
+    }
+    return $absolute_filename if $needle_exists;
+    log_error "Needle file $relative_needle_path not found within $needles_dir.";
+    return undef;
+}

lib/OpenQA/Schema/Result/Needles.pm

@@ -16,7 +16,7 @@ use OpenQA::App;
 use OpenQA::Git;
 use OpenQA::Jobs::Constants;
 use OpenQA::Schema::Result::Jobs;
-use OpenQA::Utils qw(locate_needle);
+use OpenQA::Needles qw(locate_needle);
 
 __PACKAGE__->table('needles');
 __PACKAGE__->load_components(qw(InflateColumn::DateTime Timestamps));

lib/OpenQA/Setup.pm

@@ -69,6 +69,8 @@ sub read_config ($app) {
             do_cleanup => 'no',
             git_auto_clone => 'yes',
             git_auto_update => 'no',
+            checkout_needles_sha => 'no',
+            temp_needle_refs_retention => 120,
         },
         scheduler => {
             max_job_scheduled_time => 7,

lib/OpenQA/Shared/Plugin/Gru.pm

@@ -31,7 +31,10 @@ sub register_tasks ($self) {
       OpenQA::Task::Asset::Download
       OpenQA::Task::Asset::Limit
       OpenQA::Task::Git::Clone
-      OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete
+      OpenQA::Task::Needle::Scan
+      OpenQA::Task::Needle::Save
+      OpenQA::Task::Needle::Delete
+      OpenQA::Task::Needle::LimitTempRefs
       OpenQA::Task::Job::Limit
       OpenQA::Task::Job::ArchiveResults
       OpenQA::Task::Job::FinalizeResults

lib/OpenQA/Task/Needle/LimitTempRefs.pm

@@ -0,0 +1,39 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+package OpenQA::Task::Needle::LimitTempRefs;
+use Mojo::Base 'Mojolicious::Plugin', -signatures;
+
+use File::Find;
+use File::stat;
+use Fcntl qw(S_ISDIR);
+use OpenQA::Needles;
+use OpenQA::Task::SignalGuard;
+use Time::Seconds;
+
+my $retention;
+
+sub register ($self, $app, $job) {
+    $retention = $app->config->{'scm git'}->{temp_needle_refs_retention} * ONE_MINUTE;
+    $app->minion->add_task(limit_temp_needle_refs => sub ($job) { _limit($app, $job) });
+}
+
+sub _limit ($app, $job) {
+    my $ensure_task_retry_on_termination_signal_guard = OpenQA::Task::SignalGuard->new($job);
+
+    return $job->finish({error => 'Another job to remove needle versions is running. Try again later.'})
+      unless my $guard = $app->minion->guard('limit_needle_versions_task', 7200);
+
+    # remove all temporary needles which haven't been accessed in time period specified in config
+    my $temp_dir = OpenQA::Needles::temp_dir;
+    return undef unless -d $temp_dir;
+    my $now = time;
+    my $wanted = sub {
+        return undef unless my $lstat = lstat $File::Find::name;
+        return rmdir $File::Find::name if S_ISDIR($lstat->mode);    # remove all empty dirs
+        return unlink $File::Find::name if ($now - $lstat->mtime) > $retention;
+    };
+    find({no_chdir => 1, bydepth => 1, wanted => $wanted}, $temp_dir);
+}
+
+1;

lib/OpenQA/Utils.pm

@@ -26,6 +26,7 @@ use OpenQA::Log qw(log_info log_debug log_warning log_error);
 use Config::Tiny;
 use Time::HiRes qw(tv_interval);
 use File::Basename;
+use File::Path qw(make_path);
 use File::Spec;
 use File::Spec::Functions qw(catfile catdir);
 use Fcntl;
@@ -92,7 +93,6 @@ our @EXPORT = qw(
   BUGREF_REGEX
   LABEL_REGEX
   FLAG_REGEX
-  locate_needle
   needledir
   productdir
   testcasedir
@@ -254,22 +254,6 @@ sub is_in_tests {
 
 sub needledir { productdir(@_) . '/needles' }
 
-sub locate_needle {
-    my ($relative_needle_path, $needles_dir) = @_;
-
-    my $absolute_filename = catdir($needles_dir, $relative_needle_path);
-    my $needle_exists = -f $absolute_filename;
-
-    if (!$needle_exists) {
-        $absolute_filename = catdir(sharedir(), $relative_needle_path);
-        $needle_exists = -f $absolute_filename;
-    }
-    return $absolute_filename if $needle_exists;
-
-    log_error("Needle file $relative_needle_path not found within $needles_dir.");
-    return undef;
-}
-
 # Adds a timestamp to a string (eg. needle name) or replace the already present timestamp
 sub ensure_timestamp_appended {
     my ($str) = @_;
@@ -333,10 +317,12 @@ sub run_cmd_with_log {
 sub run_cmd_with_log_return_error ($cmd, %args) {
     my $stdout_level = $args{stdout} // 'debug';
     my $stderr_level = $args{stderr} // 'debug';
+    my $output_file = $args{output_file};
     log_info('Running cmd: ' . join(' ', @$cmd));
     try {
         my ($stdin, $stdout_err, $stdout, $stderr) = ('') x 4;
-        my $ipc_run_succeeded = IPC::Run::run($cmd, \$stdin, \$stdout, \$stderr);
+        my @out_args = defined $output_file ? ('>', $output_file, '2>', \$stderr) : (\$stdout, \$stderr);
+        my $ipc_run_succeeded = IPC::Run::run($cmd, \$stdin, @out_args);
         my $return_code = $?;
         chomp $stderr;
         if ($ipc_run_succeeded) {

lib/OpenQA/WebAPI/Controller/File.pm

@@ -6,6 +6,7 @@
 
 BEGIN { $ENV{MAGICK_THREAD_LIMIT} = 1; }
 
+use OpenQA::Needles;
 use OpenQA::Utils qw(:DEFAULT prjdir assetdir imagesdir);
 use File::Basename;
 use File::Spec;
@@ -29,11 +30,22 @@
 
     # make sure the directory of the file parameter is a real subdir of testcasedir before
     # using it to find needle subdirectory, to prevent access outside of the zoo
-    if ($jsonfile && !is_in_tests($jsonfile)) {
+    if ($jsonfile && !is_in_tests($jsonfile) && !OpenQA::Needles::is_in_temp_dir($jsonfile)) {
         my $prjdir = prjdir();
         warn "$jsonfile is not in a subdir of $prjdir/share/tests or $prjdir/tests";
         return $self->render(text => 'Forbidden', status => 403);
     }
+    # If the json file in not in the tests we may be using a temporary
+    # directory for needles from a different git SHA
+    my $jsonfile_in_temp_dir = $jsonfile && OpenQA::Needles::is_in_temp_dir($jsonfile);
+    if ($jsonfile_in_temp_dir) {
+        $needledir = dirname($jsonfile);
+        # In case we're in a subdirectory, keep taking the dirname until we
+        # have the path of the `needles` directory
+        while (basename($needledir) ne 'needles') {
+            $needledir = dirname($needledir);
+        }
+    }
     # Reject directory traversal breakouts here...
     if (index($jsonfile, '..') != -1) {
         warn "jsonfile value $jsonfile is invalid, cannot contain ..";

lib/OpenQA/WebAPI/Controller/Step.pm

@@ -9,9 +9,12 @@
 use Mojo::File 'path';
 use Mojo::URL;
 use Mojo::Util 'decode';
-use OpenQA::Utils qw(ensure_timestamp_appended find_bug_number locate_needle needledir testcasedir);
+use OpenQA::Needles qw(needle_temp_dir locate_needle);
+use OpenQA::Utils qw(ensure_timestamp_appended find_bug_number needledir testcasedir
+  run_cmd_with_log run_cmd_with_log_return_error);
 use OpenQA::Jobs::Constants;
 use File::Basename;
+use File::Path 'make_path';
 use File::Which 'which';
 use POSIX 'strftime';
 use Mojo::JSON 'decode_json';
@@ -91,6 +94,34 @@
     $self->viewimg;
 }
 
+sub _determine_needles_dir_for_job ($self, $job) {
+    return undef unless $self->app->config->{'scm git'}->{checkout_needles_sha} eq 'yes';
+    return undef unless my $needle_dirs = realpath($job->needle_dir);
+    my $settings = $job->settings;
+    return ($needle_dirs, $settings->single({key => 'NEEDLES_DIR'}) // $settings->single({key => 'CASEDIR'}));
+}
+
+sub _create_tmpdir_for_needles_refspec ($self, $job) {
+    my ($needle_dirs, $needles_dir_var) = $self->_determine_needles_dir_for_job($job);
+    return undef unless $needles_dir_var;
+    my $needles_url = Mojo::URL->new($needles_dir_var->value);
+    return undef unless $needles_url->scheme;
+    my $needles_ref = $needles_url->fragment;
+    eval {
+        my $vars = decode_json(path($job->result_dir, 'vars.json')->slurp);
+        $needles_ref = $vars->{NEEDLES_GIT_HASH} if ref $vars eq 'HASH';
+    };
+    chomp $needles_ref;
+    return undef unless $needles_ref;
+    return undef unless run_cmd_with_log ['git', '-C', $needle_dirs, 'fetch', '--depth', 1, 'origin', $needles_ref];
+    my $rev_parse_res = run_cmd_with_log_return_error ['git', '-C', $needle_dirs, 'rev-parse', 'FETCH_HEAD'];
+    return undef unless $rev_parse_res->{status};
+    $needles_ref = $rev_parse_res->{stdout};
+    chomp $needles_ref;
+    needle_temp_dir($needle_dirs, $needles_ref)->make_path;
+    return $needles_ref;
+}
+
 # Needle editor
 sub edit ($self) {
     return $self->reply->not_found unless $self->_init && $self->check_tabmode();
@@ -101,6 +132,7 @@
     my $distri = $job->DISTRI;
     my $dversion = $job->VERSION || '';
     my $needle_dir = $job->needle_dir;
+    my $needle_ref = $self->_create_tmpdir_for_needles_refspec($job);
     my $app = $self->app;
     my $needles_rs = $app->schema->resultset('Needles');
 
@@ -130,7 +162,7 @@
         # Second position: the only needle (with the same matches)
         my $needle_info
           = $self->_extended_needle_info($needle_dir, $needle_name, \%basic_needle_data, $module_detail->{json},
-            0, \@error_messages);
+            0, \@error_messages, $needle_ref);
         if ($needle_info) {
             $needle_info->{matches} = $screenshot->{matches};
             push(@needles, $needle_info);
@@ -144,10 +176,10 @@
         # $needle contains information from result, in which 'areas' refers to the best matches.
         # We also use $area for transforming the match information into a real area
         for my $needle (@$module_detail_needles) {
-            my $needle_info = $self->_extended_needle_info(
-                $needle_dir, $needle->{name}, \%basic_needle_data,
-                $needle->{json}, $needle->{error}, \@error_messages
-            ) || next;
+            my $needle_info
+              = $self->_extended_needle_info($needle_dir, $needle->{name}, \%basic_needle_data,
+                $needle->{json}, $needle->{error}, \@error_messages, $needle_ref)
+              || next;
             my $matches = $needle_info->{matches};
             for my $match (@{$needle->{area}}) {
                 my %area = (
@@ -188,7 +220,7 @@
             # get needle info to show the needle also in selection
             my $needle_info
               = $self->_extended_needle_info($needle_dir, $new_needle->name, \%basic_needle_data, $new_needle->path,
-                undef, \@error_messages)
+                undef, \@error_messages, $needle_ref)
               || next;
             $needle_info->{title} = 'new: ' . $needle_info->{title};
             push(@needles, $needle_info);
@@ -274,9 +306,9 @@
     return \%screenshot;
 }
 
-sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_dir) {
+sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_dir, $needle_ref) {
     $file_name //= "$name.json";
-    $file_name = locate_needle($file_name, $needles_dir) if !-f $file_name;
+    $file_name = locate_needle($file_name, $needles_dir, $needle_ref) if !-f $file_name;
     return (undef, 'File not found') unless defined $file_name;
 
     my $needle;
@@ -303,11 +335,14 @@
     return ($needle, undef);
 }
 
-sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $file_name, $error, $error_messages) {
+sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $file_name, $error, $error_messages,
+    $needle_ref)
+{
     my $overall_list_of_tags = $basic_needle_data->{tags};
     my $distri = $basic_needle_data->{distri};
     my $version = $basic_needle_data->{version};
-    my ($needle_info, $err) = $self->_basic_needle_info($needle_name, $distri, $version, $file_name, $needle_dir);
+    my ($needle_info, $err)
+      = $self->_basic_needle_info($needle_name, $distri, $version, $file_name, $needle_dir, $needle_ref);
     unless (defined $needle_info) {
         push(@$error_messages, "Could not parse needle $needle_name for $distri $version: $err");
         return undef;
@@ -467,6 +502,7 @@
     my $distri = $job->DISTRI;
     my $dversion = $job->VERSION || '';
     my $needle_dir = $job->needle_dir;
+    my $needle_ref = $self->_create_tmpdir_for_needles_refspec($job);
     my $real_needle_dir = realpath($needle_dir) // $needle_dir;
     my $needles_rs = $self->app->schema->resultset('Needles');
 
@@ -502,7 +538,8 @@
     # load primary needle match
     my $primary_match;
     if (my $needle = $module_detail->{needle}) {
-        my ($needleinfo) = $self->_basic_needle_info($needle, $distri, $dversion, $module_detail->{json}, $needle_dir);
+        my ($needleinfo)
+          = $self->_basic_needle_info($needle, $distri, $dversion, $module_detail->{json}, $needle_dir, $needle_ref);
         if ($needleinfo) {
             my $info = {
                 name => $needle,
@@ -524,7 +561,8 @@
     if ($module_detail->{needles}) {
         for my $needle (@{$module_detail->{needles}}) {
             my $needlename = $needle->{name};
-            my ($needleinfo) = $self->_basic_needle_info($needlename, $distri, $dversion, $needle->{json}, $needle_dir);
+            my ($needleinfo)
+              = $self->_basic_needle_info($needlename, $distri, $dversion, $needle->{json}, $needle_dir, $needle_ref);
             next unless $needleinfo;
             my $info = {
                 name => $needlename,

script/openqa-enqueue-needle-ref-cleanup

@@ -0,0 +1,2 @@
+#!/bin/sh -e
+exec "$(dirname "$0")"/openqa eval -m production -V 'app->gru->enqueue(limit_temp_needle_refs => [], {priority => 5, ttl => 172800, limit => 1})' "$@"