Skip to content

Commit

Permalink
Add cleanup for alternate needle versions
Browse files Browse the repository at this point in the history
There is now a minion task which will clean up alternate needle files
created by the WebUI. The minimum retention time of the needle files can
be set in the config, and defaults to 30 minutes
  • Loading branch information
sdclarke committed Nov 24, 2023
1 parent d926009 commit bbe3787
Show file tree
Hide file tree
Showing 6 changed files with 44 additions and 3 deletions.
1 change: 1 addition & 0 deletions lib/OpenQA/Setup.pm
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ sub read_config ($app) {
do_push => 'no',
do_cleanup => 'no',
checkout_needles_sha => 'no',
minimum_needle_retention_time => undef,
},
'scheduler' => {
max_job_scheduled_time => 7,
Expand Down
2 changes: 1 addition & 1 deletion lib/OpenQA/Shared/Plugin/Gru.pm
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ sub register_tasks ($self) {
for (
qw(OpenQA::Task::AuditEvents::Limit),
qw(OpenQA::Task::Asset::Download OpenQA::Task::Asset::Limit),
qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete),
qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete OpenQA::Task::Needle::RemoveVersions),
qw(OpenQA::Task::Job::Limit),
qw(OpenQA::Task::Job::ArchiveResults),
qw(OpenQA::Task::Job::FinalizeResults),
Expand Down
38 changes: 38 additions & 0 deletions lib/OpenQA/Task/Needle/RemoveVersions.pm
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Copyright SUSE LLC
# SPDX-License-Identifier: GPL-2.0-or-later

package OpenQA::Task::Needle::RemoveVersions;
use Mojo::Base 'Mojolicious::Plugin', -signatures;

use File::Find;
use File::stat;

my $minimum_retention_time;

sub register ($self, $app, $job) {
$minimum_retention_time = $app->config->{'scm git'}->{minimum_needle_retention_time} * 60 // 30 * 60;
$app->minion->add_task(remove_needle_versions => sub ($job) { _remove_needle_versions($app, $job) });
}

sub _remove_needle_versions ($app, $job) {
return $job->finish({error => 'Another job to remove needle versions is running. Try again later.'})
unless my $guard = $app->minion->guard('limit_needle_versions_task', 7200);

my $needle_versions_path = "/tmp/needle_dirs";
return unless -d $needle_versions_path;
# Remove all temporary needles which haven't been accessed in time period specified in config
find(\&wanted, $needle_versions_path);
return;
}

sub wanted() {
my $filepath = $File::Find::name;
return unless -f $filepath;
my $now = time;
my $atime = stat($filepath)->atime;
if (($now - $atime) > $minimum_retention_time) {
unlink($filepath);
}
}

1;
2 changes: 1 addition & 1 deletion lib/OpenQA/Utils.pm
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ sub locate_needle {

if ($needles_ref) {
my $needles_dir_basename = basename(dirname($needles_dir));
my $temp_needles_dir = "/tmp/$needles_dir_basename/worktrees/$needles_ref/needles";
my $temp_needles_dir = "/tmp/needle_dirs/$needles_dir_basename/$needles_ref/needles";
if (File::Spec->splitdir($relative_needle_path) > 1) {
make_path($temp_needles_dir . '/' . dirname($relative_needle_path));
}
Expand Down
2 changes: 1 addition & 1 deletion lib/OpenQA/WebAPI/Controller/Step.pm
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ sub _create_tmpdir_for_needles_refspec ($self, $job) {
chomp($needles_ref);
return undef unless $needles_ref;
my $needle_dir_basename = basename(dirname($needle_dir));
my $new_path = "/tmp/$needle_dir_basename/worktrees/$needles_ref/needles";
my $new_path = "/tmp/needle_dirs/$needle_dir_basename/$needles_ref/needles";
if (!-d $new_path) {
make_path($new_path);
qx{git -C "$needle_dir" fetch --depth 1 origin "$needles_ref" &>/dev/null};
Expand Down
2 changes: 2 additions & 0 deletions script/openqa-enqueue-needle-versions-cleanup
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#!/bin/sh -e
exec "$(dirname "$0")"/openqa eval -m production -V 'app->gru->enqueue(remove_needle_versions => [], {priority => 5, ttl => 172800, limit => 1})' "$@"

0 comments on commit bbe3787

Please sign in to comment.