Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(notification): send notification to admin when job failed #3653

Closed
wants to merge 14 commits into from
Closed
Prev Previous commit
chore: remove unnecessary exit_code
Signed-off-by: Wei Zhang <kweizh@tabbyml.com>
zwpaper committed Jan 15, 2025
commit e8d8d3f5e4b0363cf61eb979197fb11ef7eb2d95
19 changes: 10 additions & 9 deletions ee/tabby-webserver/src/service/background_job/db.rs
Original file line number Diff line number Diff line change
@@ -20,19 +20,19 @@
context: Arc<dyn ContextService>,
db: DbConn,
) -> tabby_schema::Result<()> {
let mut has_error = false;

Check warning on line 23 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L23

Added line #L23 was not covered by tests

if let Err(e) = db.delete_expired_token().await {
has_error = true;
logkit::warn!("Failed to delete expired tokens: {}", e);

Check warning on line 27 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L26-L27

Added lines #L26 - L27 were not covered by tests
};
if let Err(e) = db.delete_expired_password_resets().await {
has_error = true;
logkit::warn!("Failed to delete expired password resets: {}", e);

Check warning on line 31 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L30-L31

Added lines #L30 - L31 were not covered by tests
};
if let Err(e) = db.delete_expired_ephemeral_threads().await {
has_error = true;
logkit::warn!("Failed to delete expired ephemeral threads: {}", e);

Check warning on line 35 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L34-L35

Added lines #L34 - L35 were not covered by tests
};

// Read all active sources
@@ -47,56 +47,57 @@
.delete_unused_source_id_read_access_policy(&active_source_ids)
.await
{
has_error = true;
logkit::warn!(

Check warning on line 51 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L50-L51

Added lines #L50 - L51 were not covered by tests
"Failed to delete unused source id read access policy: {}",
e
);
};
}
Err(e) => {
has_error = true;
logkit::warn!("Failed to read active sources: {}", e);

Check warning on line 59 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L58-L59

Added lines #L58 - L59 were not covered by tests
}
}

if let Err(e) = Self::data_retention(now, &db).await {
has_error = true;
logkit::warn!("Failed to run data retention job: {}", e);

Check warning on line 65 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L64-L65

Added lines #L64 - L65 were not covered by tests
}

if has_error {
if !has_error {

Check warning on line 68 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L68

Added line #L68 was not covered by tests
Ok(())
} else {
Err(CoreError::Other(anyhow::anyhow!(
"Failed to run db maintenance job"

Check warning on line 72 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L72

Added line #L72 was not covered by tests
)))
}
}

async fn data_retention(now: DateTime<Utc>, db: &DbConn) -> tabby_schema::Result<()> {
let mut errors = vec![];
let mut has_error = false;

if let Err(e) = db.delete_job_run_before_three_months(now).await {
errors.push(format!(
has_error = true;
logkit::warn!(

Check warning on line 82 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L81-L82

Added lines #L81 - L82 were not covered by tests
"Failed to clean up and retain only the last 3 months of jobs: {}",
e
));
);
}

if let Err(e) = db.delete_user_events_before_three_months(now).await {
errors.push(format!(
has_error = true;
logkit::warn!(

Check warning on line 90 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L89-L90

Added lines #L89 - L90 were not covered by tests
"Failed to clean up and retain only the last 3 months of user events: {}",
e
));
);
}

if errors.is_empty() {
if !has_error {
Ok(())
} else {
Err(CoreError::Other(anyhow::anyhow!(
"{}",
errors.join(";\n\n")
"Failed to run data retention job"

Check warning on line 100 in ee/tabby-webserver/src/service/background_job/db.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/db.rs#L100

Added line #L100 was not covered by tests
)))
}
}
Original file line number Diff line number Diff line change
@@ -22,11 +22,11 @@
license_service: Arc<dyn LicenseService>,
notification_service: Arc<dyn NotificationService>,
) -> tabby_schema::Result<()> {
let license = match license_service.read().await {
Ok(license) => license,
Err(err) => {
logkit::warn!(exit_code = -1; "Failed to read license: {}", err);
logkit::warn!("Failed to read license: {}", err);
return Err(err);

Check warning on line 29 in ee/tabby-webserver/src/service/background_job/license_check.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/license_check.rs#L25-L29

Added lines #L25 - L29 were not covered by tests
}
};
if license.r#type == LicenseType::Community {
@@ -34,20 +34,20 @@
}
if let Some(expire_in_days) = license.expire_in_days() {
if expire_in_days < 7 && expire_in_days > 0 {
if let Err(e) = notification_service

Check warning on line 37 in ee/tabby-webserver/src/service/background_job/license_check.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/license_check.rs#L37

Added line #L37 was not covered by tests
.create(
NotificationRecipient::Admin,
&make_expring_message(expire_in_days),
)
.await

Check warning on line 42 in ee/tabby-webserver/src/service/background_job/license_check.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/license_check.rs#L42

Added line #L42 was not covered by tests
{
logkit::warn!(exit_code = -1; "Failed to create notification: {}", e);
logkit::warn!("Failed to create notification: {}", e);
return Err(e);
}
}
}

Ok(())

Check warning on line 50 in ee/tabby-webserver/src/service/background_job/license_check.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/license_check.rs#L44-L50

Added lines #L44 - L50 were not covered by tests
}
}

4 changes: 2 additions & 2 deletions ee/tabby-webserver/src/service/background_job/mod.rs
Original file line number Diff line number Diff line change
@@ -49,16 +49,16 @@
BackgroundJobEvent::SchedulerGitRepository(repository) => {
write!(f, "SyncGitRepository::{}", repository.git_url)
}
BackgroundJobEvent::SchedulerGithubGitlabRepository(integration_id) => {
write!(f, "SyncGithubGitlabRepository::{}", integration_id)

Check warning on line 53 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L52-L53

Added lines #L52 - L53 were not covered by tests
}
BackgroundJobEvent::SyncThirdPartyRepositories(integration_id) => {
write!(f, "SyncThirdPartyRepositories::{}", integration_id)

Check warning on line 56 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L55-L56

Added lines #L55 - L56 were not covered by tests
}
BackgroundJobEvent::WebCrawler(job) => write!(f, "WebCrawler::{}", job.url()),
BackgroundJobEvent::IndexGarbageCollection => write!(f, "IndexGarbageCollection"),

Check warning on line 59 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L58-L59

Added lines #L58 - L59 were not covered by tests
}
}

Check warning on line 61 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L61

Added line #L61 was not covered by tests
}

impl BackgroundJobEvent {
@@ -98,9 +98,9 @@
),
)
.await
.map_err(|err| {
warn!("Failed to send notification: {:?}", err);
})

Check warning on line 103 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L101-L103

Added lines #L101 - L103 were not covered by tests
.ok();
}};
}
@@ -138,16 +138,16 @@
continue;
}

let job_id = job.id;
let logger = JobLogger::new(db.clone(), job_id);
debug!("Background job {} started, command: {}", job_id, job.command);

Check warning on line 143 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L141-L143

Added lines #L141 - L143 were not covered by tests
let Ok(event) = serde_json::from_str::<BackgroundJobEvent>(&job.command) else {
logkit::info!(exit_code = -1; "Failed to parse background job event, marking it as failed");
continue;
};

let job_name = event.to_string();
let result = match event {

Check warning on line 150 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L149-L150

Added lines #L149 - L150 were not covered by tests
BackgroundJobEvent::SchedulerGitRepository(repository_config) => {
let job = SchedulerGitJob::new(repository_config);
job.run(embedding.clone()).await
@@ -170,105 +170,105 @@
};
debug!("Background job {} completed", job.id);

match &result {
Err(err) => {
logkit::info!(exit_code = 1; "Job failed {}", err);
logkit::warn!(exit_code = 1; "Job failed {}", err);
logger.finalize().await;
notify_job_error!(notification_service, err, job_name, job_id);

Check warning on line 177 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L173-L177

Added lines #L173 - L177 were not covered by tests
},
_ => {
logkit::info!(exit_code = 0; "Job completed successfully");
logger.finalize().await;

Check warning on line 181 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L180-L181

Added lines #L180 - L181 were not covered by tests
}
}
},
Some(now) = hourly.next() => {
run_job(
&db,
DbMaintainanceJob.name(),
&DbMaintainanceJob.to_command(),
|| DbMaintainanceJob::cron(now, context_service.clone(), db.clone()),
notification_service.clone(),
).await;

Check warning on line 192 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L186-L192

Added lines #L186 - L192 were not covered by tests


run_job(
&db,
SchedulerGitJob::NAME,
"cron",
|| SchedulerGitJob::cron(now, git_repository_service.clone(), job_service.clone()),
notification_service.clone(),
).await;

Check warning on line 201 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L195-L201

Added lines #L195 - L201 were not covered by tests

run_job(
&db,
SyncIntegrationJob::NAME,
"cron",
|| SyncIntegrationJob::cron(now, integration_service.clone(), job_service.clone()),
notification_service.clone(),
).await;

Check warning on line 209 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L203-L209

Added lines #L203 - L209 were not covered by tests

run_job(
&db,
SchedulerGithubGitlabJob::NAME,
"cron",
|| SchedulerGithubGitlabJob::cron(now, third_party_repository_service.clone(), job_service.clone()),
notification_service.clone(),
).await;

Check warning on line 217 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L211-L217

Added lines #L211 - L217 were not covered by tests

run_job(
&db,
IndexGarbageCollection.name(),
&IndexGarbageCollection.to_command(),
|| IndexGarbageCollection.run(repository_service.clone(), context_service.clone()),
notification_service.clone(),
).await;

Check warning on line 225 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L219-L225

Added lines #L219 - L225 were not covered by tests
},
Some(now) = daily.next() => {
run_job(
&db,
LicenseCheckJob::NAME,
"cron",
|| LicenseCheckJob::cron(now, license_service.clone(), notification_service.clone()),
notification_service.clone(),
).await;

Check warning on line 234 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L228-L234

Added lines #L228 - L234 were not covered by tests
}
else => {
warn!("Background job channel closed");
return;
break;
}
};
}
});
}

async fn run_job<F, Fut>(
db: &DbConn,
job_name: &str,
command: &str,
job_fn: F,
notification_service: Arc<dyn NotificationService>,
) where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = tabby_schema::Result<()>>,
{
let job_id = match db
.create_job_run(job_name.to_string(), command.to_string())
.await

Check warning on line 257 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L245-L257

Added lines #L245 - L257 were not covered by tests
{
Ok(job_id) => job_id,
Err(err) => {
warn!("Failed to create job run for {}: {}", job_name, err);
return;

Check warning on line 262 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L259-L262

Added lines #L259 - L262 were not covered by tests
}
};

let logger = JobLogger::new(db.clone(), job_id);
if let Err(err) = job_fn().await {
logkit::warn!(exit_code = 1; "Job failed {}", err);
notify_job_error!(notification_service, err, job_name, job_id);

Check warning on line 269 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L266-L269

Added lines #L266 - L269 were not covered by tests
} else {
logkit::info!(exit_code = 0; "Job completed successfully");

Check warning on line 271 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L271

Added line #L271 was not covered by tests
}
logger.finalize().await;
}

Check warning on line 274 in ee/tabby-webserver/src/service/background_job/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/mod.rs#L273-L274

Added lines #L273 - L274 were not covered by tests
Original file line number Diff line number Diff line change
@@ -55,18 +55,18 @@
job: Arc<dyn JobService>,
) -> tabby_schema::Result<()> {
debug!("Syncing all github and gitlab repositories");
let integrations = match integration

Check warning on line 58 in ee/tabby-webserver/src/service/background_job/third_party_integration.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/third_party_integration.rs#L58

Added line #L58 was not covered by tests
.list_integrations(None, None, None, None, None, None)
.await

Check warning on line 60 in ee/tabby-webserver/src/service/background_job/third_party_integration.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/third_party_integration.rs#L60

Added line #L60 was not covered by tests
{
Ok(integrations) => integrations,
Err(err) => {
logkit::warn!(exit_code = -1; "Failed to list integrations: {}", err);
logkit::warn!("Failed to list integrations: {}", err);
return Err(err);

Check warning on line 65 in ee/tabby-webserver/src/service/background_job/third_party_integration.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/third_party_integration.rs#L62-L65

Added lines #L62 - L65 were not covered by tests
}
};

for integration in integrations {

Check warning on line 69 in ee/tabby-webserver/src/service/background_job/third_party_integration.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/third_party_integration.rs#L69

Added line #L69 was not covered by tests
let _ = job
.trigger(
BackgroundJobEvent::SyncThirdPartyRepositories(integration.id).to_command(),
@@ -237,18 +237,18 @@
repository: Arc<dyn ThirdPartyRepositoryService>,
job: Arc<dyn JobService>,
) -> tabby_schema::Result<()> {
let repositories = match repository

Check warning on line 240 in ee/tabby-webserver/src/service/background_job/third_party_integration.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/third_party_integration.rs#L240

Added line #L240 was not covered by tests
.list_repositories_with_filter(None, None, Some(true), None, None, None, None)
.await

Check warning on line 242 in ee/tabby-webserver/src/service/background_job/third_party_integration.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/third_party_integration.rs#L242

Added line #L242 was not covered by tests
{
Ok(repos) => repos,
Err(err) => {
logkit::warn!(exit_code = -1; "Failed to list repositories: {}", err);
logkit::warn!("Failed to list repositories: {}", err);
return Err(err);

Check warning on line 247 in ee/tabby-webserver/src/service/background_job/third_party_integration.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/third_party_integration.rs#L244-L247

Added lines #L244 - L247 were not covered by tests
}
};

for repository in repositories {

Check warning on line 251 in ee/tabby-webserver/src/service/background_job/third_party_integration.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/third_party_integration.rs#L251

Added line #L251 was not covered by tests
let _ = job
.trigger(
BackgroundJobEvent::SchedulerGithubGitlabRepository(repository.id).to_command(),

Unchanged files with check annotations Beta

git_repository: Arc<dyn GitRepositoryService>,
job: Arc<dyn JobService>,
) -> tabby_schema::Result<()> {
let repositories = match git_repository.repository_list().await {
Ok(repos) => repos,
Err(err) => {
logkit::warn!("Failed to list repositories: {}", err);
return Err(err);

Check warning on line 49 in ee/tabby-webserver/src/service/background_job/git.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/git.rs#L45-L49

Added lines #L45 - L49 were not covered by tests
}
};
pub trait Job: serde::Serialize {
const NAME: &'static str;
fn name(&self) -> &'static str {
Self::NAME
}
fn to_command(&self) -> String {
serde_json::to_string(self).unwrap()
}

Check warning on line 15 in ee/tabby-webserver/src/service/background_job/helper/mod.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/helper/mod.rs#L10-L15

Added lines #L10 - L15 were not covered by tests
}
context: Arc<dyn ContextService>,
) -> tabby_schema::Result<()> {
// Run garbage collection on the index
let sources = match context.read(None).await {
Ok(sources) => sources,
Err(err) => {
logkit::warn!("Failed to list sources: {}", err);
return Err(err);

Check warning on line 27 in ee/tabby-webserver/src/service/background_job/index_garbage_collection.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/index_garbage_collection.rs#L23-L27

Added lines #L23 - L27 were not covered by tests
}
};
let sources = sources

Check warning on line 30 in ee/tabby-webserver/src/service/background_job/index_garbage_collection.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/index_garbage_collection.rs#L30

Added line #L30 was not covered by tests
.sources
.into_iter()
.map(|x| x.source_id())
.collect::<Vec<_>>();
if let Err(e) = run_index_garbage_collection(sources) {
logkit::warn!("Failed to run index garbage collection: {}", e);
return Err(e.into());
}

Check warning on line 39 in ee/tabby-webserver/src/service/background_job/index_garbage_collection.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/index_garbage_collection.rs#L36-L39

Added lines #L36 - L39 were not covered by tests
// Run garbage collection on the code repositories (cloned directories)
let repositories = match repository.list_all_code_repository().await {
Ok(repos) => repos,
Err(err) => {
logkit::warn!("Failed to list repositories: {}", err);
return Err(err);

Check warning on line 46 in ee/tabby-webserver/src/service/background_job/index_garbage_collection.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/index_garbage_collection.rs#L42-L46

Added lines #L42 - L46 were not covered by tests
}
};
let mut code = CodeIndexer::default();
}
}
pub fn url(&self) -> &str {
&self.url
}

Check warning on line 39 in ee/tabby-webserver/src/service/background_job/web_crawler.rs

Codecov / codecov/patch

ee/tabby-webserver/src/service/background_job/web_crawler.rs#L37-L39

Added lines #L37 - L39 were not covered by tests
pub async fn run_impl(self, embedding: Arc<dyn Embedding>) -> tabby_schema::Result<()> {
logkit::info!("Starting doc index pipeline for {}", self.url);