From 2bdf709b63d31b6625ee2de7ec7f19e98f2234d1 Mon Sep 17 00:00:00 2001 From: Scott Clarke Date: Thu, 1 Jun 2023 11:36:19 +0100 Subject: [PATCH 1/6] Use needles from correct ref of NEEDLES_DIR If the NEEDLES_DIR (or CASEDIR as a backup) variable is a git repo URL with a fragment specifying the ref of the repository, we attempt to use it as the source of needles when viewing needle diffs in test results. If the NEEDLES_GIT_SHA variable is set, we use that instead as it should be the most accurate way to determine exactly which commit was used --- lib/OpenQA/Setup.pm | 1 + lib/OpenQA/Utils.pm | 23 ++++++++++- lib/OpenQA/WebAPI/Controller/File.pm | 14 ++++++- lib/OpenQA/WebAPI/Controller/Step.pm | 59 ++++++++++++++++++++++------ t/config.t | 1 + 5 files changed, 83 insertions(+), 15 deletions(-) diff --git a/lib/OpenQA/Setup.pm b/lib/OpenQA/Setup.pm index bedfe749b06..5096d65bafa 100644 --- a/lib/OpenQA/Setup.pm +++ b/lib/OpenQA/Setup.pm @@ -69,6 +69,7 @@ sub read_config ($app) { do_cleanup => 'no', git_auto_clone => 'yes', git_auto_update => 'no', + checkout_needles_sha => 'no', }, scheduler => { max_job_scheduled_time => 7, diff --git a/lib/OpenQA/Utils.pm b/lib/OpenQA/Utils.pm index d59bcdd605c..97057ea2b87 100644 --- a/lib/OpenQA/Utils.pm +++ b/lib/OpenQA/Utils.pm @@ -26,6 +26,7 @@ use OpenQA::Log qw(log_info log_debug log_warning log_error); use Config::Tiny; use Time::HiRes qw(tv_interval); use File::Basename; +use File::Path qw(make_path); use File::Spec; use File::Spec::Functions qw(catfile catdir); use Fcntl; @@ -255,7 +256,27 @@ sub is_in_tests { sub needledir { productdir(@_) . '/needles' } sub locate_needle { - my ($relative_needle_path, $needles_dir) = @_; + my ($relative_needle_path, $needles_dir, $needles_ref) = @_; + + if ($needles_ref) { + my $needles_dir_basename = basename(dirname($needles_dir)); + my $temp_needles_dir = "/tmp/needle_dirs/$needles_dir_basename/$needles_ref/needles"; + if (File::Spec->splitdir($relative_needle_path) > 1) { + make_path($temp_needles_dir . '/' . dirname($relative_needle_path)); + } + my $temp_json_path = $temp_needles_dir . '/' . $relative_needle_path; + open my $temp_json_fh, '>', $temp_json_path; + my $jsonfile = qx{git -C $needles_dir show $needles_ref:./$relative_needle_path}; + print $temp_json_fh $jsonfile; + close $temp_json_fh; + my $png_name = dirname($relative_needle_path) . '/' . basename($relative_needle_path, '.json') . '.png'; + my $temp_png_path = $temp_needles_dir . '/' . $png_name; + open my $temp_png_fh, '>', $temp_png_path; + my $pngfile = qx{git -C $needles_dir show $needles_ref:./$png_name}; + print $temp_png_fh $pngfile; + close $temp_png_fh; + return $temp_json_path if $? == 0; + } my $absolute_filename = catdir($needles_dir, $relative_needle_path); my $needle_exists = -f $absolute_filename; diff --git a/lib/OpenQA/WebAPI/Controller/File.pm b/lib/OpenQA/WebAPI/Controller/File.pm index 91a867916f8..d3fb38ef769 100644 --- a/lib/OpenQA/WebAPI/Controller/File.pm +++ b/lib/OpenQA/WebAPI/Controller/File.pm @@ -29,11 +29,23 @@ sub needle ($self) { # make sure the directory of the file parameter is a real subdir of testcasedir before # using it to find needle subdirectory, to prevent access outside of the zoo - if ($jsonfile && !is_in_tests($jsonfile)) { + # Also allow the json file to be under /tmp + if ($jsonfile && !is_in_tests($jsonfile) && index($jsonfile, '/tmp') != 0) { my $prjdir = prjdir(); warn "$jsonfile is not in a subdir of $prjdir/share/tests or $prjdir/tests"; return $self->render(text => 'Forbidden', status => 403); } + # If the json file in not in the tests we may be using a temporary + # directory for needles from a different git SHA + # Allow only if the jsonfile is under /tmp + if (!is_in_tests($jsonfile) && index($jsonfile, '/tmp') == 0) { + $needledir = dirname($jsonfile); + # In case we're in a subdirectory, keep taking the dirname until we + # have the path of the `needles` directory + while (basename($needledir) ne 'needles') { + $needledir = dirname($needledir); + } + } # Reject directory traversal breakouts here... if (index($jsonfile, '..') != -1) { warn "jsonfile value $jsonfile is invalid, cannot contain .."; diff --git a/lib/OpenQA/WebAPI/Controller/Step.pm b/lib/OpenQA/WebAPI/Controller/Step.pm index 4542ff72941..c7290736be1 100644 --- a/lib/OpenQA/WebAPI/Controller/Step.pm +++ b/lib/OpenQA/WebAPI/Controller/Step.pm @@ -12,6 +12,7 @@ use Mojo::Util 'decode'; use OpenQA::Utils qw(ensure_timestamp_appended find_bug_number locate_needle needledir testcasedir); use OpenQA::Jobs::Constants; use File::Basename; +use File::Path 'make_path'; use File::Which 'which'; use POSIX 'strftime'; use Mojo::JSON 'decode_json'; @@ -91,6 +92,31 @@ sub view ($self) { $self->viewimg; } +sub _create_tmpdir_for_needles_refspec ($self, $job) { + return undef unless $self->app->config->{'scm git'}->{checkout_needles_sha} eq 'yes'; + my $needle_dir = $job->needle_dir; + $needle_dir = realpath($needle_dir) // $needle_dir; + my $needles_dir_var = $job->settings->single({key => 'NEEDLES_DIR'}); + $needles_dir_var = $job->settings->single({key => 'CASEDIR'}) unless $needles_dir_var; + return undef unless $needles_dir_var; + my $needles_url = Mojo::URL->new($needles_dir_var->value); + return undef unless $needles_url->scheme; + my $needles_ref = $needles_url->fragment; + eval { + my $vars_json = Mojo::File->new($job->result_dir(), 'vars.json')->slurp; + my $vars = decode_json($vars_json); + $needles_ref = $vars->{NEEDLES_GIT_HASH}; + }; + chomp($needles_ref); + return undef unless $needles_ref; + qx{git -C "$needle_dir" fetch --depth 1 origin "$needles_ref" &>/dev/null}; + $needles_ref = qx{git -C "$needle_dir" rev-parse FETCH_HEAD}; + my $needle_dir_basename = basename(dirname($needle_dir)); + my $new_path = "/tmp/needle_dirs/$needle_dir_basename/$needles_ref/needles"; + make_path($new_path); + return $needles_ref; +} + # Needle editor sub edit ($self) { return $self->reply->not_found unless $self->_init && $self->check_tabmode(); @@ -101,6 +127,7 @@ sub edit ($self) { my $distri = $job->DISTRI; my $dversion = $job->VERSION || ''; my $needle_dir = $job->needle_dir; + my $needle_ref = $self->_create_tmpdir_for_needles_refspec($job); my $app = $self->app; my $needles_rs = $app->schema->resultset('Needles'); @@ -130,7 +157,7 @@ sub edit ($self) { # Second position: the only needle (with the same matches) my $needle_info = $self->_extended_needle_info($needle_dir, $needle_name, \%basic_needle_data, $module_detail->{json}, - 0, \@error_messages); + 0, \@error_messages, $needle_ref); if ($needle_info) { $needle_info->{matches} = $screenshot->{matches}; push(@needles, $needle_info); @@ -144,10 +171,10 @@ sub edit ($self) { # $needle contains information from result, in which 'areas' refers to the best matches. # We also use $area for transforming the match information into a real area for my $needle (@$module_detail_needles) { - my $needle_info = $self->_extended_needle_info( - $needle_dir, $needle->{name}, \%basic_needle_data, - $needle->{json}, $needle->{error}, \@error_messages - ) || next; + my $needle_info + = $self->_extended_needle_info($needle_dir, $needle->{name}, \%basic_needle_data, + $needle->{json}, $needle->{error}, \@error_messages, $needle_ref) + || next; my $matches = $needle_info->{matches}; for my $match (@{$needle->{area}}) { my %area = ( @@ -188,7 +215,7 @@ sub edit ($self) { # get needle info to show the needle also in selection my $needle_info = $self->_extended_needle_info($needle_dir, $new_needle->name, \%basic_needle_data, $new_needle->path, - undef, \@error_messages) + undef, \@error_messages, $needle_ref) || next; $needle_info->{title} = 'new: ' . $needle_info->{title}; push(@needles, $needle_info); @@ -274,9 +301,9 @@ sub _new_screenshot ($self, $tags, $image_name, $matches = undef) { return \%screenshot; } -sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_dir) { +sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_dir, $needle_ref) { $file_name //= "$name.json"; - $file_name = locate_needle($file_name, $needles_dir) if !-f $file_name; + $file_name = locate_needle($file_name, $needles_dir, $needle_ref) if !-f $file_name; return (undef, 'File not found') unless defined $file_name; my $needle; @@ -303,11 +330,14 @@ sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_di return ($needle, undef); } -sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $file_name, $error, $error_messages) { +sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $file_name, $error, $error_messages, + $needle_ref) +{ my $overall_list_of_tags = $basic_needle_data->{tags}; my $distri = $basic_needle_data->{distri}; my $version = $basic_needle_data->{version}; - my ($needle_info, $err) = $self->_basic_needle_info($needle_name, $distri, $version, $file_name, $needle_dir); + my ($needle_info, $err) + = $self->_basic_needle_info($needle_name, $distri, $version, $file_name, $needle_dir, $needle_ref); unless (defined $needle_info) { push(@$error_messages, "Could not parse needle $needle_name for $distri $version: $err"); return undef; @@ -467,6 +497,7 @@ sub viewimg ($self) { my $distri = $job->DISTRI; my $dversion = $job->VERSION || ''; my $needle_dir = $job->needle_dir; + my $needle_ref = $self->_create_tmpdir_for_needles_refspec($job); my $real_needle_dir = realpath($needle_dir) // $needle_dir; my $needles_rs = $self->app->schema->resultset('Needles'); @@ -479,7 +510,7 @@ sub viewimg ($self) { my $append_needle_info = sub ($tags, $needle_info) { # add timestamps and URLs from database $self->populate_hash_with_needle_timestamps_and_urls( - $needles_rs->find_needle($real_needle_dir, "$needle_info->{name}.json"), $needle_info); + $needles_rs->find_needle($job->needle_dir, "$needle_info->{name}.json"), $needle_info); # handle case when the needle has (for some reason) no tags if (!$tags) { @@ -502,7 +533,8 @@ sub viewimg ($self) { # load primary needle match my $primary_match; if (my $needle = $module_detail->{needle}) { - my ($needleinfo) = $self->_basic_needle_info($needle, $distri, $dversion, $module_detail->{json}, $needle_dir); + my ($needleinfo) + = $self->_basic_needle_info($needle, $distri, $dversion, $module_detail->{json}, $needle_dir, $needle_ref); if ($needleinfo) { my $info = { name => $needle, @@ -524,7 +556,8 @@ sub viewimg ($self) { if ($module_detail->{needles}) { for my $needle (@{$module_detail->{needles}}) { my $needlename = $needle->{name}; - my ($needleinfo) = $self->_basic_needle_info($needlename, $distri, $dversion, $needle->{json}, $needle_dir); + my ($needleinfo) + = $self->_basic_needle_info($needlename, $distri, $dversion, $needle->{json}, $needle_dir, $needle_ref); next unless $needleinfo; my $info = { name => $needlename, diff --git a/t/config.t b/t/config.t index 43eb4cb3422..262a0781e93 100644 --- a/t/config.t +++ b/t/config.t @@ -67,6 +67,7 @@ subtest 'Test configuration default modes' => sub { do_cleanup => 'no', git_auto_clone => 'yes', git_auto_update => 'no', + checkout_needles_sha => 'no', }, 'scheduler' => { max_job_scheduled_time => 7, From 87b9fdcc1cc66a2e983026259aed5c90150e1c52 Mon Sep 17 00:00:00 2001 From: Scott Clarke Date: Thu, 8 Jun 2023 16:25:18 +0100 Subject: [PATCH 2/6] Add cleanup for alternate needle versions There is now a minion task which will clean up alternate needle files created by the WebUI. The minimum retention time of the needle files can be set in the config, and defaults to 30 minutes --- lib/OpenQA/Setup.pm | 1 + lib/OpenQA/Shared/Plugin/Gru.pm | 1 + lib/OpenQA/Task/Needle/RemoveVersions.pm | 38 +++++++++++++++++++ script/openqa-enqueue-needle-versions-cleanup | 2 + 4 files changed, 42 insertions(+) create mode 100644 lib/OpenQA/Task/Needle/RemoveVersions.pm create mode 100755 script/openqa-enqueue-needle-versions-cleanup diff --git a/lib/OpenQA/Setup.pm b/lib/OpenQA/Setup.pm index 5096d65bafa..014db05c9da 100644 --- a/lib/OpenQA/Setup.pm +++ b/lib/OpenQA/Setup.pm @@ -70,6 +70,7 @@ sub read_config ($app) { git_auto_clone => 'yes', git_auto_update => 'no', checkout_needles_sha => 'no', + minimum_needle_retention_time => undef, }, scheduler => { max_job_scheduled_time => 7, diff --git a/lib/OpenQA/Shared/Plugin/Gru.pm b/lib/OpenQA/Shared/Plugin/Gru.pm index 2c47963b323..b9be28c7454 100644 --- a/lib/OpenQA/Shared/Plugin/Gru.pm +++ b/lib/OpenQA/Shared/Plugin/Gru.pm @@ -32,6 +32,7 @@ sub register_tasks ($self) { OpenQA::Task::Asset::Limit OpenQA::Task::Git::Clone OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete + qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete OpenQA::Task::Needle::RemoveVersions), OpenQA::Task::Job::Limit OpenQA::Task::Job::ArchiveResults OpenQA::Task::Job::FinalizeResults diff --git a/lib/OpenQA/Task/Needle/RemoveVersions.pm b/lib/OpenQA/Task/Needle/RemoveVersions.pm new file mode 100644 index 00000000000..3ad61b457e7 --- /dev/null +++ b/lib/OpenQA/Task/Needle/RemoveVersions.pm @@ -0,0 +1,38 @@ +# Copyright SUSE LLC +# SPDX-License-Identifier: GPL-2.0-or-later + +package OpenQA::Task::Needle::RemoveVersions; +use Mojo::Base 'Mojolicious::Plugin', -signatures; + +use File::Find; +use File::stat; + +my $minimum_retention_time; + +sub register ($self, $app, $job) { + $minimum_retention_time = ($app->config->{'scm git'}->{minimum_needle_retention_time} // 30) * 60; + $app->minion->add_task(remove_needle_versions => sub ($job) { _remove_needle_versions($app, $job) }); +} + +sub _remove_needle_versions ($app, $job) { + return $job->finish({error => 'Another job to remove needle versions is running. Try again later.'}) + unless my $guard = $app->minion->guard('limit_needle_versions_task', 7200); + + my $needle_versions_path = "/tmp/needle_dirs"; + return unless -d $needle_versions_path; + # Remove all temporary needles which haven't been accessed in time period specified in config + find(\&wanted, $needle_versions_path); + return; +} + +sub wanted () { + my $filepath = $File::Find::name; + return unless -f $filepath; + my $now = time; + my $atime = stat($filepath)->atime; + if (($now - $atime) > $minimum_retention_time) { + unlink($filepath); + } +} + +1; diff --git a/script/openqa-enqueue-needle-versions-cleanup b/script/openqa-enqueue-needle-versions-cleanup new file mode 100755 index 00000000000..84f71de4a24 --- /dev/null +++ b/script/openqa-enqueue-needle-versions-cleanup @@ -0,0 +1,2 @@ +#!/bin/sh -e +exec "$(dirname "$0")"/openqa eval -m production -V 'app->gru->enqueue(remove_needle_versions => [], {priority => 5, ttl => 172800, limit => 1})' "$@" From 60bdc81585afcdea9e5b58e5e3342e70c57576a1 Mon Sep 17 00:00:00 2001 From: Scott Clarke Date: Thu, 1 Feb 2024 13:16:39 +0000 Subject: [PATCH 3/6] Add unit test for needle version removal --- t/14-grutasks.t | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/t/14-grutasks.t b/t/14-grutasks.t index 840b147a36b..4912bee2ddf 100644 --- a/t/14-grutasks.t +++ b/t/14-grutasks.t @@ -24,6 +24,7 @@ use OpenQA::Test::Case; use File::Which 'which'; use File::Path (); use Mojo::Util qw(dumper scope_guard); +use File::Touch (); use Date::Format 'time2str'; use Fcntl ':mode'; use Mojo::File qw(path tempdir); @@ -410,6 +411,19 @@ subtest 'limit_results_and_logs gru task cleans up logs' => sub { ok !-e $log_file_for_groupless_job, 'log file for groupless job got cleaned'; }; +subtest 'remove_needle_versions gru task cleans up needle versions' => sub { + # Create a temporary needle file older than the configured expiry time (defaults to 30 minutes) + my $temp_needle_path = '/tmp/needle_dirs/test_repo/branch/needles'; + File::Path->make_path($temp_needle_path); + my @needle_files = ($temp_needle_path . 'needle.png', $temp_needle_path . 'needle.json'); + my $ref = File::Touch->new(atime => time - (30 * 60 + 1)); + $ref->touch(@needle_files); + + # Run cleanup + run_gru_job $t->app, 'remove_needle_versions'; + ok !-e $_ for @needle_files; +}; + subtest 'limit audit events' => sub { my $app = $t->app; my $audit_events = $app->schema->resultset('AuditEvents'); From 52367a6a9a22e4aa56aed81d54fa8137f2896320 Mon Sep 17 00:00:00 2001 From: Marius Kittler Date: Thu, 7 Mar 2024 13:10:23 +0100 Subject: [PATCH 4/6] Improve code for showing correct version of needles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Improve error handling * Simplify code * Split code into smaller functions * Move needle-related code into its own module so utilities don't become too big * Avoid using a shell to run Git commands; this breaks when needle paths contain characters that needed escaping * Avoid invoking Git commands if temporary needle files are still present * Use the usual directory the web UI stores cache files in (instead of hard-coding `/tmp/…`) * Output temporary files directly instead of using intermediate buffer * Keep using the real needle path as before for populating last seen/match because using the relative path breaks compatibility with existing databases and this change is supposedly not required anyway * See https://progress.opensuse.org/issues/154783 --- etc/openqa/openqa.ini | 2 + lib/OpenQA/Git.pm | 9 +++++ lib/OpenQA/Needles.pm | 55 ++++++++++++++++++++++++++++ lib/OpenQA/Schema/Result/Needles.pm | 2 +- lib/OpenQA/Utils.pm | 41 ++------------------- lib/OpenQA/WebAPI/Controller/File.pm | 8 ++-- lib/OpenQA/WebAPI/Controller/Step.pm | 37 +++++++++++-------- 7 files changed, 95 insertions(+), 59 deletions(-) create mode 100644 lib/OpenQA/Needles.pm diff --git a/etc/openqa/openqa.ini b/etc/openqa/openqa.ini index 785ec5c6bc9..90ba042b532 100644 --- a/etc/openqa/openqa.ini +++ b/etc/openqa/openqa.ini @@ -117,6 +117,8 @@ #git_auto_clone = yes ## Experimental - Ensure a git update of all test code and needles #git_auto_update = no +# whether openQA should attempt to display needles of the correct version in the web UI +#checkout_needles_sha = no ## Authentication method to use for user management [auth] diff --git a/lib/OpenQA/Git.pm b/lib/OpenQA/Git.pm index 40b6d2f97df..7f455d79398 100644 --- a/lib/OpenQA/Git.pm +++ b/lib/OpenQA/Git.pm @@ -145,4 +145,13 @@ sub is_workdir_clean ($self) { return $r->{status}; } +sub cache_ref ($self, $ref, $relative_path, $output_file) { + return undef if -f $output_file; + my @git = $self->_prepare_git_command; + my $res = run_cmd_with_log_return_error [@git, 'show', "$ref:./$relative_path"], output_file => $output_file; + return undef if $res->{status}; + unlink $output_file; + return _format_git_error($res, 'Unable to cache Git ref'); +} + 1; diff --git a/lib/OpenQA/Needles.pm b/lib/OpenQA/Needles.pm new file mode 100644 index 00000000000..fce601a2819 --- /dev/null +++ b/lib/OpenQA/Needles.pm @@ -0,0 +1,55 @@ +# Copyright SUSE LLC +# SPDX-License-Identifier: GPL-2.0-or-later + +package OpenQA::Needles; +use Mojo::Base -strict, -signatures; + +use Exporter qw(import); +use File::Basename; +use File::Spec; +use File::Spec::Functions qw(catdir); +use OpenQA::Git; +use OpenQA::Log qw(log_error); +use OpenQA::Utils qw(prjdir sharedir); +use Mojo::File qw(path); + +our @EXPORT = qw(is_in_temp_dir needle_temp_dir locate_needle); + +my $tmp_dir = prjdir() . '/webui/cache/needle-refs'; + +sub is_in_temp_dir ($file_path) { index($file_path, $tmp_dir) == 0 } + +sub needle_temp_dir ($dir, $ref) { path($tmp_dir, basename(dirname($dir)), $ref, 'needles') } + +sub _locate_needle_for_ref ($relative_needle_path, $needles_dir, $needles_ref) { + return undef unless defined $needles_ref; + + my $temp_needles_dir = needle_temp_dir($needles_dir, $needles_ref); + my $subdir = dirname($relative_needle_path); + path($temp_needles_dir, $subdir)->make_path if File::Spec->splitdir($relative_needle_path) > 1; + + my $git = OpenQA::Git->new(dir => $needles_dir); + my $temp_json_path = "$temp_needles_dir/$relative_needle_path"; + my $basename = basename($relative_needle_path, '.json'); + my $relative_png_path = "$subdir/$basename.png"; + my $temp_png_path = "$temp_needles_dir/$relative_png_path"; + my $error = $git->cache_ref($needles_ref, $relative_needle_path, $temp_json_path) + // $git->cache_ref($needles_ref, $relative_png_path, $temp_png_path); + return $temp_json_path unless defined $error; + log_error "An error occurred when looking for ref '$needles_ref' of '$relative_needle_path': $error"; + return undef; +} + +sub locate_needle ($relative_needle_path, $needles_dir, $needles_ref = undef) { + my $location_for_ref = _locate_needle_for_ref($relative_needle_path, $needles_dir, $needles_ref); + return $location_for_ref if defined $location_for_ref; + my $absolute_filename = catdir($needles_dir, $relative_needle_path); + my $needle_exists = -f $absolute_filename; + if (!$needle_exists) { + $absolute_filename = catdir(sharedir(), $relative_needle_path); + $needle_exists = -f $absolute_filename; + } + return $absolute_filename if $needle_exists; + log_error "Needle file $relative_needle_path not found within $needles_dir."; + return undef; +} diff --git a/lib/OpenQA/Schema/Result/Needles.pm b/lib/OpenQA/Schema/Result/Needles.pm index 5af2980cc67..8ae89ab5463 100644 --- a/lib/OpenQA/Schema/Result/Needles.pm +++ b/lib/OpenQA/Schema/Result/Needles.pm @@ -16,7 +16,7 @@ use OpenQA::App; use OpenQA::Git; use OpenQA::Jobs::Constants; use OpenQA::Schema::Result::Jobs; -use OpenQA::Utils qw(locate_needle); +use OpenQA::Needles qw(locate_needle); __PACKAGE__->table('needles'); __PACKAGE__->load_components(qw(InflateColumn::DateTime Timestamps)); diff --git a/lib/OpenQA/Utils.pm b/lib/OpenQA/Utils.pm index 97057ea2b87..67704616610 100644 --- a/lib/OpenQA/Utils.pm +++ b/lib/OpenQA/Utils.pm @@ -93,7 +93,6 @@ our @EXPORT = qw( BUGREF_REGEX LABEL_REGEX FLAG_REGEX - locate_needle needledir productdir testcasedir @@ -255,42 +254,6 @@ sub is_in_tests { sub needledir { productdir(@_) . '/needles' } -sub locate_needle { - my ($relative_needle_path, $needles_dir, $needles_ref) = @_; - - if ($needles_ref) { - my $needles_dir_basename = basename(dirname($needles_dir)); - my $temp_needles_dir = "/tmp/needle_dirs/$needles_dir_basename/$needles_ref/needles"; - if (File::Spec->splitdir($relative_needle_path) > 1) { - make_path($temp_needles_dir . '/' . dirname($relative_needle_path)); - } - my $temp_json_path = $temp_needles_dir . '/' . $relative_needle_path; - open my $temp_json_fh, '>', $temp_json_path; - my $jsonfile = qx{git -C $needles_dir show $needles_ref:./$relative_needle_path}; - print $temp_json_fh $jsonfile; - close $temp_json_fh; - my $png_name = dirname($relative_needle_path) . '/' . basename($relative_needle_path, '.json') . '.png'; - my $temp_png_path = $temp_needles_dir . '/' . $png_name; - open my $temp_png_fh, '>', $temp_png_path; - my $pngfile = qx{git -C $needles_dir show $needles_ref:./$png_name}; - print $temp_png_fh $pngfile; - close $temp_png_fh; - return $temp_json_path if $? == 0; - } - - my $absolute_filename = catdir($needles_dir, $relative_needle_path); - my $needle_exists = -f $absolute_filename; - - if (!$needle_exists) { - $absolute_filename = catdir(sharedir(), $relative_needle_path); - $needle_exists = -f $absolute_filename; - } - return $absolute_filename if $needle_exists; - - log_error("Needle file $relative_needle_path not found within $needles_dir."); - return undef; -} - # Adds a timestamp to a string (eg. needle name) or replace the already present timestamp sub ensure_timestamp_appended { my ($str) = @_; @@ -354,10 +317,12 @@ sub run_cmd_with_log { sub run_cmd_with_log_return_error ($cmd, %args) { my $stdout_level = $args{stdout} // 'debug'; my $stderr_level = $args{stderr} // 'debug'; + my $output_file = $args{output_file}; log_info('Running cmd: ' . join(' ', @$cmd)); try { my ($stdin, $stdout_err, $stdout, $stderr) = ('') x 4; - my $ipc_run_succeeded = IPC::Run::run($cmd, \$stdin, \$stdout, \$stderr); + my @out_args = defined $output_file ? ('>', $output_file, '2>', \$stderr) : (\$stdout, \$stderr); + my $ipc_run_succeeded = IPC::Run::run($cmd, \$stdin, @out_args); my $return_code = $?; chomp $stderr; if ($ipc_run_succeeded) { diff --git a/lib/OpenQA/WebAPI/Controller/File.pm b/lib/OpenQA/WebAPI/Controller/File.pm index d3fb38ef769..67658f69dc7 100644 --- a/lib/OpenQA/WebAPI/Controller/File.pm +++ b/lib/OpenQA/WebAPI/Controller/File.pm @@ -6,6 +6,7 @@ use Mojo::Base 'Mojolicious::Controller', -signatures; BEGIN { $ENV{MAGICK_THREAD_LIMIT} = 1; } +use OpenQA::Needles; use OpenQA::Utils qw(:DEFAULT prjdir assetdir imagesdir); use File::Basename; use File::Spec; @@ -29,16 +30,15 @@ sub needle ($self) { # make sure the directory of the file parameter is a real subdir of testcasedir before # using it to find needle subdirectory, to prevent access outside of the zoo - # Also allow the json file to be under /tmp - if ($jsonfile && !is_in_tests($jsonfile) && index($jsonfile, '/tmp') != 0) { + if ($jsonfile && !is_in_tests($jsonfile) && !OpenQA::Needles::is_in_temp_dir($jsonfile)) { my $prjdir = prjdir(); warn "$jsonfile is not in a subdir of $prjdir/share/tests or $prjdir/tests"; return $self->render(text => 'Forbidden', status => 403); } # If the json file in not in the tests we may be using a temporary # directory for needles from a different git SHA - # Allow only if the jsonfile is under /tmp - if (!is_in_tests($jsonfile) && index($jsonfile, '/tmp') == 0) { + my $jsonfile_in_temp_dir = $jsonfile && OpenQA::Needles::is_in_temp_dir($jsonfile); + if ($jsonfile_in_temp_dir) { $needledir = dirname($jsonfile); # In case we're in a subdirectory, keep taking the dirname until we # have the path of the `needles` directory diff --git a/lib/OpenQA/WebAPI/Controller/Step.pm b/lib/OpenQA/WebAPI/Controller/Step.pm index c7290736be1..f51dc2564fb 100644 --- a/lib/OpenQA/WebAPI/Controller/Step.pm +++ b/lib/OpenQA/WebAPI/Controller/Step.pm @@ -9,7 +9,9 @@ use Encode 'decode_utf8'; use Mojo::File 'path'; use Mojo::URL; use Mojo::Util 'decode'; -use OpenQA::Utils qw(ensure_timestamp_appended find_bug_number locate_needle needledir testcasedir); +use OpenQA::Needles qw(needle_temp_dir locate_needle); +use OpenQA::Utils qw(ensure_timestamp_appended find_bug_number needledir testcasedir + run_cmd_with_log run_cmd_with_log_return_error); use OpenQA::Jobs::Constants; use File::Basename; use File::Path 'make_path'; @@ -92,28 +94,31 @@ sub view ($self) { $self->viewimg; } -sub _create_tmpdir_for_needles_refspec ($self, $job) { +sub _determine_needles_dir_for_job ($self, $job) { return undef unless $self->app->config->{'scm git'}->{checkout_needles_sha} eq 'yes'; - my $needle_dir = $job->needle_dir; - $needle_dir = realpath($needle_dir) // $needle_dir; - my $needles_dir_var = $job->settings->single({key => 'NEEDLES_DIR'}); - $needles_dir_var = $job->settings->single({key => 'CASEDIR'}) unless $needles_dir_var; + return undef unless my $needle_dirs = realpath($job->needle_dir); + my $settings = $job->settings; + return ($needle_dirs, $settings->single({key => 'NEEDLES_DIR'}) // $settings->single({key => 'CASEDIR'})); +} + +sub _create_tmpdir_for_needles_refspec ($self, $job) { + my ($needle_dirs, $needles_dir_var) = $self->_determine_needles_dir_for_job($job); return undef unless $needles_dir_var; my $needles_url = Mojo::URL->new($needles_dir_var->value); return undef unless $needles_url->scheme; my $needles_ref = $needles_url->fragment; eval { - my $vars_json = Mojo::File->new($job->result_dir(), 'vars.json')->slurp; - my $vars = decode_json($vars_json); - $needles_ref = $vars->{NEEDLES_GIT_HASH}; + my $vars = decode_json(path($job->result_dir, 'vars.json')->slurp); + $needles_ref = $vars->{NEEDLES_GIT_HASH} if ref $vars eq 'HASH'; }; - chomp($needles_ref); + chomp $needles_ref; return undef unless $needles_ref; - qx{git -C "$needle_dir" fetch --depth 1 origin "$needles_ref" &>/dev/null}; - $needles_ref = qx{git -C "$needle_dir" rev-parse FETCH_HEAD}; - my $needle_dir_basename = basename(dirname($needle_dir)); - my $new_path = "/tmp/needle_dirs/$needle_dir_basename/$needles_ref/needles"; - make_path($new_path); + return undef unless run_cmd_with_log ['git', '-C', $needle_dirs, 'fetch', '--depth', 1, 'origin', $needles_ref]; + my $rev_parse_res = run_cmd_with_log_return_error ['git', '-C', $needle_dirs, 'rev-parse', 'FETCH_HEAD']; + return undef unless $rev_parse_res->{status}; + $needles_ref = $rev_parse_res->{stdout}; + chomp $needles_ref; + needle_temp_dir($needle_dirs, $needles_ref)->make_path; return $needles_ref; } @@ -510,7 +515,7 @@ sub viewimg ($self) { my $append_needle_info = sub ($tags, $needle_info) { # add timestamps and URLs from database $self->populate_hash_with_needle_timestamps_and_urls( - $needles_rs->find_needle($job->needle_dir, "$needle_info->{name}.json"), $needle_info); + $needles_rs->find_needle($real_needle_dir, "$needle_info->{name}.json"), $needle_info); # handle case when the needle has (for some reason) no tags if (!$tags) { From 124af5cc971b6335422ee9bf49406901acc3e81e Mon Sep 17 00:00:00 2001 From: Marius Kittler Date: Mon, 11 Mar 2024 18:18:14 +0100 Subject: [PATCH 5/6] Improve cleanup of temporary needle refs * Use the settings key `temp_needle_refs_retention` which makes it clear that this setting is about temporary needle refs * Document settings key in default `openqa.ini` * Adapt the cleanup to be in accordance with the changed path for temporary needle refs * Rename cleanup task to `limit_temp_needle_refs` to be more in-line with existing cleanup tasks * Add systemd units to enqueue the cleanup task automatically * Change the default retention to two hours (from 30 minutes) * Use the modification time instead of the access time because the access time might be updated very to easily unintendedly * Use signal guard to retry cleanup in case the gru service is restarted * See https://progress.opensuse.org/issues/154783 --- etc/openqa/openqa.ini | 2 + lib/OpenQA/Git.pm | 6 ++- lib/OpenQA/Needles.pm | 4 +- lib/OpenQA/Setup.pm | 2 +- lib/OpenQA/Shared/Plugin/Gru.pm | 6 ++- lib/OpenQA/Task/Needle/LimitTempRefs.pm | 39 +++++++++++++++++++ lib/OpenQA/Task/Needle/RemoveVersions.pm | 38 ------------------ ...anup => openqa-enqueue-needle-ref-cleanup} | 2 +- .../openqa-enqueue-needle-ref-cleanup.service | 9 +++++ .../openqa-enqueue-needle-ref-cleanup.timer | 9 +++++ t/14-grutasks.t | 35 +++++++++++------ t/config.t | 1 + 12 files changed, 98 insertions(+), 55 deletions(-) create mode 100644 lib/OpenQA/Task/Needle/LimitTempRefs.pm delete mode 100644 lib/OpenQA/Task/Needle/RemoveVersions.pm rename script/{openqa-enqueue-needle-versions-cleanup => openqa-enqueue-needle-ref-cleanup} (60%) create mode 100644 systemd/openqa-enqueue-needle-ref-cleanup.service create mode 100644 systemd/openqa-enqueue-needle-ref-cleanup.timer diff --git a/etc/openqa/openqa.ini b/etc/openqa/openqa.ini index 90ba042b532..20ad30ad8c9 100644 --- a/etc/openqa/openqa.ini +++ b/etc/openqa/openqa.ini @@ -119,6 +119,8 @@ #git_auto_update = no # whether openQA should attempt to display needles of the correct version in the web UI #checkout_needles_sha = no +# retention for storing temporary needle refs in minutes +#temp_needle_refs_retention = 120 ## Authentication method to use for user management [auth] diff --git a/lib/OpenQA/Git.pm b/lib/OpenQA/Git.pm index 7f455d79398..96c33911d90 100644 --- a/lib/OpenQA/Git.pm +++ b/lib/OpenQA/Git.pm @@ -6,6 +6,7 @@ package OpenQA::Git; use Mojo::Base -base, -signatures; use Mojo::Util 'trim'; use Cwd 'abs_path'; +use File::Touch; use OpenQA::Utils qw(run_cmd_with_log_return_error); has 'app'; @@ -146,7 +147,10 @@ sub is_workdir_clean ($self) { } sub cache_ref ($self, $ref, $relative_path, $output_file) { - return undef if -f $output_file; + if (-f $output_file) { + eval { touch $output_file }; + return $@ ? $@ : undef; + } my @git = $self->_prepare_git_command; my $res = run_cmd_with_log_return_error [@git, 'show', "$ref:./$relative_path"], output_file => $output_file; return undef if $res->{status}; diff --git a/lib/OpenQA/Needles.pm b/lib/OpenQA/Needles.pm index fce601a2819..25a2990a111 100644 --- a/lib/OpenQA/Needles.pm +++ b/lib/OpenQA/Needles.pm @@ -13,10 +13,12 @@ use OpenQA::Log qw(log_error); use OpenQA::Utils qw(prjdir sharedir); use Mojo::File qw(path); -our @EXPORT = qw(is_in_temp_dir needle_temp_dir locate_needle); +our @EXPORT = qw(temp_dir is_in_temp_dir needle_temp_dir locate_needle); my $tmp_dir = prjdir() . '/webui/cache/needle-refs'; +sub temp_dir () { $tmp_dir } + sub is_in_temp_dir ($file_path) { index($file_path, $tmp_dir) == 0 } sub needle_temp_dir ($dir, $ref) { path($tmp_dir, basename(dirname($dir)), $ref, 'needles') } diff --git a/lib/OpenQA/Setup.pm b/lib/OpenQA/Setup.pm index 014db05c9da..36fc87478d1 100644 --- a/lib/OpenQA/Setup.pm +++ b/lib/OpenQA/Setup.pm @@ -70,7 +70,7 @@ sub read_config ($app) { git_auto_clone => 'yes', git_auto_update => 'no', checkout_needles_sha => 'no', - minimum_needle_retention_time => undef, + temp_needle_refs_retention => 120, }, scheduler => { max_job_scheduled_time => 7, diff --git a/lib/OpenQA/Shared/Plugin/Gru.pm b/lib/OpenQA/Shared/Plugin/Gru.pm index b9be28c7454..7a90460fe4a 100644 --- a/lib/OpenQA/Shared/Plugin/Gru.pm +++ b/lib/OpenQA/Shared/Plugin/Gru.pm @@ -31,8 +31,10 @@ sub register_tasks ($self) { OpenQA::Task::Asset::Download OpenQA::Task::Asset::Limit OpenQA::Task::Git::Clone - OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete - qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete OpenQA::Task::Needle::RemoveVersions), + OpenQA::Task::Needle::Scan + OpenQA::Task::Needle::Save + OpenQA::Task::Needle::Delete + OpenQA::Task::Needle::LimitTempRefs OpenQA::Task::Job::Limit OpenQA::Task::Job::ArchiveResults OpenQA::Task::Job::FinalizeResults diff --git a/lib/OpenQA/Task/Needle/LimitTempRefs.pm b/lib/OpenQA/Task/Needle/LimitTempRefs.pm new file mode 100644 index 00000000000..cfbce1eb7ba --- /dev/null +++ b/lib/OpenQA/Task/Needle/LimitTempRefs.pm @@ -0,0 +1,39 @@ +# Copyright SUSE LLC +# SPDX-License-Identifier: GPL-2.0-or-later + +package OpenQA::Task::Needle::LimitTempRefs; +use Mojo::Base 'Mojolicious::Plugin', -signatures; + +use File::Find; +use File::stat; +use Fcntl qw(S_ISDIR); +use OpenQA::Needles; +use OpenQA::Task::SignalGuard; +use Time::Seconds; + +my $retention; + +sub register ($self, $app, $job) { + $retention = $app->config->{'scm git'}->{temp_needle_refs_retention} * ONE_MINUTE; + $app->minion->add_task(limit_temp_needle_refs => sub ($job) { _limit($app, $job) }); +} + +sub _limit ($app, $job) { + my $ensure_task_retry_on_termination_signal_guard = OpenQA::Task::SignalGuard->new($job); + + return $job->finish({error => 'Another job to remove needle versions is running. Try again later.'}) + unless my $guard = $app->minion->guard('limit_needle_versions_task', 7200); + + # remove all temporary needles which haven't been accessed in time period specified in config + my $temp_dir = OpenQA::Needles::temp_dir; + return undef unless -d $temp_dir; + my $now = time; + my $wanted = sub { + return undef unless my $lstat = lstat $File::Find::name; + return rmdir $File::Find::name if S_ISDIR($lstat->mode); # remove all empty dirs + return unlink $File::Find::name if ($now - $lstat->mtime) > $retention; + }; + find({no_chdir => 1, bydepth => 1, wanted => $wanted}, $temp_dir); +} + +1; diff --git a/lib/OpenQA/Task/Needle/RemoveVersions.pm b/lib/OpenQA/Task/Needle/RemoveVersions.pm deleted file mode 100644 index 3ad61b457e7..00000000000 --- a/lib/OpenQA/Task/Needle/RemoveVersions.pm +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright SUSE LLC -# SPDX-License-Identifier: GPL-2.0-or-later - -package OpenQA::Task::Needle::RemoveVersions; -use Mojo::Base 'Mojolicious::Plugin', -signatures; - -use File::Find; -use File::stat; - -my $minimum_retention_time; - -sub register ($self, $app, $job) { - $minimum_retention_time = ($app->config->{'scm git'}->{minimum_needle_retention_time} // 30) * 60; - $app->minion->add_task(remove_needle_versions => sub ($job) { _remove_needle_versions($app, $job) }); -} - -sub _remove_needle_versions ($app, $job) { - return $job->finish({error => 'Another job to remove needle versions is running. Try again later.'}) - unless my $guard = $app->minion->guard('limit_needle_versions_task', 7200); - - my $needle_versions_path = "/tmp/needle_dirs"; - return unless -d $needle_versions_path; - # Remove all temporary needles which haven't been accessed in time period specified in config - find(\&wanted, $needle_versions_path); - return; -} - -sub wanted () { - my $filepath = $File::Find::name; - return unless -f $filepath; - my $now = time; - my $atime = stat($filepath)->atime; - if (($now - $atime) > $minimum_retention_time) { - unlink($filepath); - } -} - -1; diff --git a/script/openqa-enqueue-needle-versions-cleanup b/script/openqa-enqueue-needle-ref-cleanup similarity index 60% rename from script/openqa-enqueue-needle-versions-cleanup rename to script/openqa-enqueue-needle-ref-cleanup index 84f71de4a24..bad166b2c6b 100755 --- a/script/openqa-enqueue-needle-versions-cleanup +++ b/script/openqa-enqueue-needle-ref-cleanup @@ -1,2 +1,2 @@ #!/bin/sh -e -exec "$(dirname "$0")"/openqa eval -m production -V 'app->gru->enqueue(remove_needle_versions => [], {priority => 5, ttl => 172800, limit => 1})' "$@" +exec "$(dirname "$0")"/openqa eval -m production -V 'app->gru->enqueue(limit_temp_needle_refs => [], {priority => 5, ttl => 172800, limit => 1})' "$@" diff --git a/systemd/openqa-enqueue-needle-ref-cleanup.service b/systemd/openqa-enqueue-needle-ref-cleanup.service new file mode 100644 index 00000000000..153ff2b435e --- /dev/null +++ b/systemd/openqa-enqueue-needle-ref-cleanup.service @@ -0,0 +1,9 @@ +[Unit] +Description=Enqueues a needle ref cleanup task for openQA. +After=postgresql.service openqa-setup-db.service +Wants=openqa-setup-db.service + +[Service] +Type=oneshot +User=geekotest +ExecStart=/usr/share/openqa/script/openqa-enqueue-needle-ref-cleanup diff --git a/systemd/openqa-enqueue-needle-ref-cleanup.timer b/systemd/openqa-enqueue-needle-ref-cleanup.timer new file mode 100644 index 00000000000..f03a96e5f94 --- /dev/null +++ b/systemd/openqa-enqueue-needle-ref-cleanup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Enqueues a needle refs task for openQA every day. + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/t/14-grutasks.t b/t/14-grutasks.t index 4912bee2ddf..d60f0ecccca 100644 --- a/t/14-grutasks.t +++ b/t/14-grutasks.t @@ -12,6 +12,7 @@ use OpenQA::Jobs::Constants; use OpenQA::JobDependencies::Constants; use OpenQA::JobGroupDefaults; use OpenQA::Schema::Result::Jobs; +use OpenQA::Needles; use File::Copy; require OpenQA::Test::Database; use OpenQA::Test::Utils qw(run_gru_job perform_minion_jobs); @@ -411,17 +412,29 @@ subtest 'limit_results_and_logs gru task cleans up logs' => sub { ok !-e $log_file_for_groupless_job, 'log file for groupless job got cleaned'; }; -subtest 'remove_needle_versions gru task cleans up needle versions' => sub { - # Create a temporary needle file older than the configured expiry time (defaults to 30 minutes) - my $temp_needle_path = '/tmp/needle_dirs/test_repo/branch/needles'; - File::Path->make_path($temp_needle_path); - my @needle_files = ($temp_needle_path . 'needle.png', $temp_needle_path . 'needle.json'); - my $ref = File::Touch->new(atime => time - (30 * 60 + 1)); - $ref->touch(@needle_files); - - # Run cleanup - run_gru_job $t->app, 'remove_needle_versions'; - ok !-e $_ for @needle_files; +subtest 'limit_temp_needle_refs task cleans up temp needle refs exceeding retention' => sub { + my $temp_dir = path(OpenQA::Needles::temp_dir); + is $temp_dir, 't/data/openqa/webui/cache/needle-refs', 'needle temp dir determined as expected'; + $temp_dir->child($_)->make_path for qw(ref1 ref2); + my @old_needle_files = ("$temp_dir/ref1/needle_old.png", "$temp_dir/ref1/needle_old.json"); + my @new_needle_files = ("$temp_dir/ref2/needle_new.png", "$temp_dir/ref2/needle_new.json"); + my $now = time; + File::Touch->new(time => $now - (120 * ONE_MINUTE + 1))->touch(@old_needle_files); + File::Touch->new(time => $now + ONE_MINUTE)->touch(@new_needle_files); + + # enqueue and run cleanup + my $minion = $t->app->minion; + my $id = $minion->enqueue('limit_temp_needle_refs'); + ok defined $id, 'job enqueued'; + perform_minion_jobs $minion; + my $job_info = $minion->job($id)->info; + + subtest 'cleanup result' => sub { + is $job_info->{state}, 'finished', 'job finished'; + ok !-e $_, "old file '$_' cleaned up" for @old_needle_files; + ok !-e "$temp_dir/ref1", 'empty directory removed'; + ok -e $_, "new file '$_' preserved" for @new_needle_files; + } or diag explain $job_info; }; subtest 'limit audit events' => sub { diff --git a/t/config.t b/t/config.t index 262a0781e93..8ebb04bde82 100644 --- a/t/config.t +++ b/t/config.t @@ -68,6 +68,7 @@ subtest 'Test configuration default modes' => sub { git_auto_clone => 'yes', git_auto_update => 'no', checkout_needles_sha => 'no', + temp_needle_refs_retention => 120, }, 'scheduler' => { max_job_scheduled_time => 7, From 64cd052b662e1c14f7e3dcc48a21932c0c38c904 Mon Sep 17 00:00:00 2001 From: Liv Dywan Date: Tue, 22 Oct 2024 21:33:26 +0200 Subject: [PATCH 6/6] Drop use of undeclared File::Touch dependency --- lib/OpenQA/Git.pm | 4 ++-- t/14-grutasks.t | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/OpenQA/Git.pm b/lib/OpenQA/Git.pm index 96c33911d90..e084993021c 100644 --- a/lib/OpenQA/Git.pm +++ b/lib/OpenQA/Git.pm @@ -6,7 +6,7 @@ package OpenQA::Git; use Mojo::Base -base, -signatures; use Mojo::Util 'trim'; use Cwd 'abs_path'; -use File::Touch; +use Mojo::File 'path'; use OpenQA::Utils qw(run_cmd_with_log_return_error); has 'app'; @@ -148,7 +148,7 @@ sub is_workdir_clean ($self) { sub cache_ref ($self, $ref, $relative_path, $output_file) { if (-f $output_file) { - eval { touch $output_file }; + eval { path($output_file)->touch }; return $@ ? $@ : undef; } my @git = $self->_prepare_git_command; diff --git a/t/14-grutasks.t b/t/14-grutasks.t index d60f0ecccca..f329d14dd39 100644 --- a/t/14-grutasks.t +++ b/t/14-grutasks.t @@ -25,7 +25,6 @@ use OpenQA::Test::Case; use File::Which 'which'; use File::Path (); use Mojo::Util qw(dumper scope_guard); -use File::Touch (); use Date::Format 'time2str'; use Fcntl ':mode'; use Mojo::File qw(path tempdir); @@ -418,9 +417,16 @@ subtest 'limit_temp_needle_refs task cleans up temp needle refs exceeding retent $temp_dir->child($_)->make_path for qw(ref1 ref2); my @old_needle_files = ("$temp_dir/ref1/needle_old.png", "$temp_dir/ref1/needle_old.json"); my @new_needle_files = ("$temp_dir/ref2/needle_new.png", "$temp_dir/ref2/needle_new.json"); - my $now = time; - File::Touch->new(time => $now - (120 * ONE_MINUTE + 1))->touch(@old_needle_files); - File::Touch->new(time => $now + ONE_MINUTE)->touch(@new_needle_files); + my $old_timestamp = time - (120 * ONE_MINUTE + 1); + my $new_timestamp = time - ONE_MINUTE + 1; + foreach my $file (@old_needle_files) { + path($file)->touch; + utime $old_timestamp, $old_timestamp, $file; + } + foreach my $file (@new_needle_files) { + path($file)->touch; + utime $new_timestamp, $new_timestamp, $file; + } # enqueue and run cleanup my $minion = $t->app->minion;