From 3c2d8a398e6a2a1ef3b33f50871712e56fcc263c Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 10 Jan 2025 16:25:23 +0800 Subject: [PATCH] chore: send notification when every job failed Signed-off-by: Wei Zhang --- .../src/service/background_job/db.rs | 7 +- .../src/service/background_job/mod.rs | 65 ++++++++----------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/ee/tabby-webserver/src/service/background_job/db.rs b/ee/tabby-webserver/src/service/background_job/db.rs index 9b0f9e673789..0eefd2b8617f 100644 --- a/ee/tabby-webserver/src/service/background_job/db.rs +++ b/ee/tabby-webserver/src/service/background_job/db.rs @@ -3,7 +3,11 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use tabby_db::DbConn; -use tabby_schema::{context::ContextService, CoreError}; +use tabby_schema::{ + context::ContextService, + notification::{NotificationRecipient, NotificationService}, + CoreError, +}; use super::helper::Job; @@ -19,6 +23,7 @@ impl DbMaintainanceJob { now: DateTime, context: Arc, db: DbConn, + notification_service: Arc, ) -> tabby_schema::Result<()> { let mut errors = vec![]; diff --git a/ee/tabby-webserver/src/service/background_job/mod.rs b/ee/tabby-webserver/src/service/background_job/mod.rs index 0a9299ea0929..45582a855521 100644 --- a/ee/tabby-webserver/src/service/background_job/mod.rs +++ b/ee/tabby-webserver/src/service/background_job/mod.rs @@ -26,6 +26,7 @@ use tabby_schema::{ license::LicenseService, notification::{NotificationRecipient, NotificationService}, repository::{GitRepositoryService, RepositoryService, ThirdPartyRepositoryService}, + AsID, }; use third_party_integration::SchedulerGithubGitlabJob; use tracing::{debug, warn}; @@ -60,11 +61,18 @@ impl BackgroundJobEvent { } } -macro_rules! append_error { - ($errors:expr, $($arg:tt)*) => {{ +#[macro_export] +macro_rules! notify_job_error { + ($notification_service:expr, $err:expr, $($arg:tt)*) => {{ let msg = format!($($arg)*); - warn!("{}", msg); - $errors.push(msg); + warn!("{}: {:?}", msg, $err); + $notification_service.create( + NotificationRecipient::Admin, + &format!( + r#"Background job failed + +{}"#, msg), + ).await.unwrap(); }}; } @@ -89,7 +97,7 @@ pub async fn start( tokio::spawn(async move { loop { - let result = tokio::select! { + tokio::select! { job = db.get_next_job_to_execute() => { let Some(job) = job else { tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; @@ -101,13 +109,15 @@ pub async fn start( continue; } - let logger = JobLogger::new(db.clone(), job.id); - debug!("Background job {} started, command: {}", job.id, job.command); + let job_id = job.id; + let logger = JobLogger::new(db.clone(), job_id); + debug!("Background job {} started, command: {}", job_id, job.command); let Ok(event) = serde_json::from_str::(&job.command) else { logkit::info!(exit_code = -1; "Failed to parse background job event, marking it as failed"); continue; }; + let job_name = format!("{:?}", event); let result = match event { BackgroundJobEvent::SchedulerGitRepository(repository_config) => { let job = SchedulerGitJob::new(repository_config); @@ -135,67 +145,48 @@ pub async fn start( Err(err) => { logkit::info!(exit_code = 1; "Job failed {}", err); logger.finalize().await; - vec![err.to_string()] + notify_job_error!(notification_service, err, r#"Job {:?} failed, +Please visit [Jobs Detail](http://localhost:8080/jobs/detail?id={}) to check the error and retry. +"#, + job_name, job_id.as_id()); }, _ => { logkit::info!(exit_code = 0; "Job completed successfully"); logger.finalize().await; - vec![] } } }, Some(now) = hourly.next() => { - let mut errors = vec![]; - if let Err(err) = DbMaintainanceJob::cron(now, context_service.clone(), db.clone()).await { - append_error!(errors, "Database maintenance failed: {:?}", err); + if let Err(err) = DbMaintainanceJob::cron(now, context_service.clone(), db.clone(), notification_service.clone()).await { + warn!("Database maintenance failed: {:?}", err); } if let Err(err) = SchedulerGitJob::cron(now, git_repository_service.clone(), job_service.clone()).await { - append_error!(errors, "Scheduler job failed: {:?}", err); + notify_job_error!(notification_service, err, "Scheduler job failed"); } if let Err(err) = SyncIntegrationJob::cron(now, integration_service.clone(), job_service.clone()).await { - append_error!(errors, "Sync integration job failed: {:?}", err); + notify_job_error!(notification_service, err, "Sync integration job failed"); } if let Err(err) = SchedulerGithubGitlabJob::cron(now, third_party_repository_service.clone(), job_service.clone()).await { - append_error!(errors, "Index issues job failed: {err:?}"); + notify_job_error!(notification_service, err, "Index issues job failed"); } if let Err(err) = IndexGarbageCollection.run(repository_service.clone(), context_service.clone()).await { - append_error!(errors, "Index garbage collection job failed: {err:?}"); + notify_job_error!(notification_service, err, "Index garbage collection job failed"); } - - errors }, Some(now) = daily.next() => { - let mut errors = vec![]; if let Err(err) = LicenseCheckJob::cron(now, license_service.clone(), notification_service.clone()).await { - append_error!(errors, "License check job failed: {err:?}"); + notify_job_error!(notification_service, err, "License check job failed"); } - - errors } else => { warn!("Background job channel closed"); return; } }; - - if !result.is_empty() { - notification_service - .create( - NotificationRecipient::Admin, - &format!( - r#"Background job failed - -{}"#, - &result.join("\n\n") - ), - ) - .await - .unwrap(); - } } }); }