diff --git a/plugins/Jobs.cpp b/plugins/Jobs.cpp index 8cd08acda..7ff609276 100644 --- a/plugins/Jobs.cpp +++ b/plugins/Jobs.cpp @@ -96,7 +96,8 @@ void BedrockPlugin_Jobs::upgradeDatabase(SQLite& db) "data TEXT NOT NULL, " "priority INTEGER NOT NULL DEFAULT " + SToStr(JOBS_DEFAULT_PRIORITY) + ", " "parentJobID INTEGER NOT NULL DEFAULT 0, " - "retryAfter TEXT NOT NULL DEFAULT \"\")", + "retryAfter TEXT NOT NULL DEFAULT \"\", " + "sequentialKey TEXT NOT NULL DEFAULT \"\")", ignore)); // verify and conditionally create indexes SASSERT(db.verifyIndex("jobsName", "jobs", "( name )", false, !BedrockPlugin_Jobs::isLive)); @@ -608,13 +609,29 @@ void BedrockJobsCommand::process(SQLite& db) } } + // If sequentialKey is set, check if there are other pending jobs with the same key. + // If so, this job should wait for them to complete before running by setting the initial state to WAITING. + const string& safeSequentialKey = SContains(job, "sequentialKey") ? SQ(job["sequentialKey"]) : SQ(""); + if (safeSequentialKey != SQ("")) { + SQResult result; + if (!db.read("SELECT 1 FROM jobs " + "WHERE sequentialKey=" + safeSequentialKey + " " + "AND state IN ('QUEUED', 'RUNQUEUED', 'RUNNING', 'WAITING') " + "LIMIT 1;", result)) { + STHROW("502 Select failed"); + } + if (!result.empty()) { + initialState = "WAITING"; + } + } + // If no data was provided, use an empty object const string& safeRetryAfter = SContains(job, "retryAfter") && !job["retryAfter"].empty() ? SQ(job["retryAfter"]) : SQ(""); // Create this new job with a new generated ID const int64_t jobIDToUse = SQLiteUtils::getRandomID(db, "jobs", "jobID"); SINFO("Next jobID: " << jobIDToUse); - if (!db.writeIdempotent("INSERT INTO jobs ( jobID, created, state, name, nextRun, repeat, data, priority, parentJobID, retryAfter ) " + if (!db.writeIdempotent("INSERT INTO jobs ( jobID, created, state, name, nextRun, repeat, data, priority, parentJobID, retryAfter, sequentialKey ) " "VALUES( " + SQ(jobIDToUse) + ", " + currentTime + ", " + @@ -625,7 +642,8 @@ void BedrockJobsCommand::process(SQLite& db) safeData + ", " + SQ(priority) + ", " + SQ(parentJobID) + ", " + - safeRetryAfter + " " + + safeRetryAfter + ", " + + safeSequentialKey + " " + " );")) { STHROW("502 insert query failed"); } @@ -1011,7 +1029,7 @@ void BedrockJobsCommand::process(SQLite& db) // Verify there is a job like this and it's running SQResult result; - if (!db.read("SELECT state, nextRun, lastRun, repeat, parentJobID, json_extract(data, '$.mockRequest'), retryAfter, json_extract(data, '$.originalNextRun') " + if (!db.read("SELECT state, nextRun, lastRun, repeat, parentJobID, json_extract(data, '$.mockRequest'), retryAfter, json_extract(data, '$.originalNextRun'), sequentialKey " "FROM jobs " "WHERE jobID=" + SQ(jobID) + ";", result)) { @@ -1029,6 +1047,7 @@ void BedrockJobsCommand::process(SQLite& db) mockRequest = result[0][5] == "1"; const string retryAfter = result[0][6]; const string originalDataNextRun = result[0][7]; + const string& sequentialKey = result[0][8]; // Make sure we're finishing a job that's actually running if (state != "RUNNING" && state != "RUNQUEUED" && !mockRequest) { @@ -1199,6 +1218,17 @@ void BedrockJobsCommand::process(SQLite& db) if (!db.read("SELECT 1 FROM jobs WHERE parentJobID != 0 AND parentJobID=" + SQ(jobID) + " LIMIT 1;").empty()) { STHROW("405 Failed to delete a job with outstanding children"); } + + // Promote the next job in sequence that is WAITING + if (!sequentialKey.empty()) { + db.writeIdempotent("UPDATE jobs SET state='QUEUED' " + "WHERE jobID = (" + " SELECT jobID FROM jobs " + " WHERE sequentialKey=" + SQ(sequentialKey) + " " + " AND state='WAITING' " + " ORDER BY created ASC " + " LIMIT 1);"); + } } } @@ -1258,12 +1288,13 @@ void BedrockJobsCommand::process(SQLite& db) // - data - Data to associate with this failed job // BedrockPlugin::verifyAttributeInt64(request, "jobID", 1); + const int64_t jobID = request.calc64("jobID"); // Verify there is a job like this and it's running SQResult result; - if (!db.read("SELECT state, nextRun, lastRun, repeat " + if (!db.read("SELECT state, nextRun, lastRun, repeat, sequentialKey " "FROM jobs " - "WHERE jobID=" + SQ(request.calc64("jobID")) + ";", + "WHERE jobID=" + SQ(jobID) + ";", result)) { STHROW("502 Select failed"); } @@ -1271,10 +1302,11 @@ void BedrockJobsCommand::process(SQLite& db) STHROW("404 No job with this jobID"); } const string& state = result[0][0]; + const string& sequentialKey = result[0][4]; // Make sure we're failing a job that's actually running or running with a retryAfter if (state != "RUNNING" && state != "RUNQUEUED") { - SINFO("Trying to fail job#" << request["jobID"] << ", but isn't RUNNING or RUNQUEUED (" << state << ")"); + SINFO("Trying to fail job#" << jobID << ", but isn't RUNNING or RUNQUEUED (" << state << ")"); STHROW("405 Can only fail RUNNING or RUNQUEUED jobs"); } @@ -1289,10 +1321,21 @@ void BedrockJobsCommand::process(SQLite& db) updateList.push_back("state='FAILED'"); // Update this job - if (!db.writeIdempotent("UPDATE jobs SET " + SComposeList(updateList) + "WHERE jobID=" + SQ(request.calc64("jobID")) + ";")) { + if (!db.writeIdempotent("UPDATE jobs SET " + SComposeList(updateList) + "WHERE jobID=" + SQ(jobID) + ";")) { STHROW("502 Fail failed"); } + // Promote the next WAITING job with the same sequentialKey + if (!sequentialKey.empty()) { + db.writeIdempotent("UPDATE jobs SET state='QUEUED' " + "WHERE jobID = (" + " SELECT jobID FROM jobs " + " WHERE sequentialKey=" + SQ(sequentialKey) + " " + " AND state='WAITING' " + " ORDER BY created ASC " + " LIMIT 1);"); + } + // Successfully processed return; } @@ -1306,32 +1349,46 @@ void BedrockJobsCommand::process(SQLite& db) // - jobID - ID of the job to delete // BedrockPlugin::verifyAttributeInt64(request, "jobID", 1); + int64_t jobID = request.calc64("jobID"); // Verify there is a job like this and it's not running SQResult result; - if (!db.read("SELECT state " + if (!db.read("SELECT state, sequentialKey " "FROM jobs " - "WHERE jobID=" + SQ(request.calc64("jobID")) + ";", + "WHERE jobID=" + SQ(jobID) + ";", result)) { STHROW("502 Select failed"); } if (result.empty()) { STHROW("404 No job with this jobID"); } - if (result[0][0] == "RUNNING") { + const string& state = result[0][0]; + const string& sequentialKey = result[0][1]; + + if (state == "RUNNING") { STHROW("405 Can't delete a RUNNING job"); } - if (result[0][0] == "PAUSED") { + if (state == "PAUSED") { STHROW("405 Can't delete a parent jobs with children running"); } // Delete the job if (!db.writeIdempotent("DELETE FROM jobs " - "WHERE jobID=" + - SQ(request.calc64("jobID")) + ";")) { + "WHERE jobID=" + SQ(jobID) + ";")) { STHROW("502 Delete failed"); } + // Promote the next WAITING job with the same sequentialKey + if (!sequentialKey.empty()) { + db.writeIdempotent("UPDATE jobs SET state='QUEUED' " + "WHERE jobID = (" + " SELECT jobID FROM jobs " + " WHERE sequentialKey=" + SQ(sequentialKey) + " " + " AND state='WAITING' " + " ORDER BY created ASC " + " LIMIT 1);"); + } + // Successfully processed return; } diff --git a/plugins/Jobs.md b/plugins/Jobs.md index 48da51f39..9e95a5721 100644 --- a/plugins/Jobs.md +++ b/plugins/Jobs.md @@ -1,11 +1,12 @@ # Bedrock::Jobs -- Rock solid job queuing Bedrock::Jobs is a plugin to the [Bedrock data foundation](../README.md) that manages a scheduled job queue. Commands include: - * **CreateJob( name, [data], [firstRun], [repeat] )** - Schedules a new job, optionally in the future, optionally to repeat. + * **CreateJob( name, [data], [firstRun], [repeat], [sequentialKey] )** - Schedules a new job, optionally in the future, optionally to repeat. * *name* - Any arbitrary string name for this job. * *data* - (optional) An arbitrary data blob to associate with this job, typically JSON encoded. * *firstRun* - (optional) The time/date on which to run this job the first time, in "YYYY-MM-DD [HH:MM:SS]" format * *repeat* - (optional) Description of how this job should repeat (see ["Repeat Syntax"](#repeat-syntax) below) + * *sequentialKey* - (optional) Jobs with the same sequentialKey run one at a time, in creation order (see ["Sequential Jobs"](#sequential-jobs) below) * **GetJob( name, [connection: wait, [timeout] ] )** - Waits for a match (if requested) and atomically dequeues exactly one job. * *name* - A pattern to match in GLOB syntax (eg, "Foo*" will get the first job whose name starts with "Foo") @@ -158,3 +159,8 @@ Confused by all the above? No problem -- there are a few "canned" patterns buil * **WEEKLY** = FINISHED, + 7 DAYS These are useful if you generally want something to happen *approximately but no greater* than the indicated frequency. + +### Sequential Jobs +Jobs with the same `sequentialKey` run one at a time, in creation order. New jobs with a key that's already active are set to `WAITING` state. When the active job completes (FinishJob/FailJob/DeleteJob), the oldest WAITING job is promoted to `QUEUED`. + +**Unsupported:** `repeat` and `parentJobID` with `sequentialKey` - these jobs don't get deleted on completion, so WAITING jobs would never be promoted. diff --git a/test/clustertest/tests/FinishJobTest.cpp b/test/clustertest/tests/FinishJobTest.cpp index 9902c9d9c..a27716771 100644 --- a/test/clustertest/tests/FinishJobTest.cpp +++ b/test/clustertest/tests/FinishJobTest.cpp @@ -23,6 +23,7 @@ struct FinishJobTest : tpunit::TestFixture TEST(FinishJobTest::hasDataDelete), TEST(FinishJobTest::hasNextRun), TEST(FinishJobTest::simpleFinishJobWithHttp), + TEST(FinishJobTest::finishJobPromotesWaitingJob), AFTER(FinishJobTest::tearDown), AFTER_CLASS(FinishJobTest::tearDownClass)) { @@ -602,4 +603,48 @@ struct FinishJobTest : tpunit::TestFixture clusterTester->getTester(0).readDB("SELECT * FROM jobs WHERE jobID = " + jobID + ";", result); ASSERT_TRUE(result.empty()); } + + // FinishJob should promote the next WAITING job with same sequentialKey + void finishJobPromotesWaitingJob() + { + // Create first job + SData command("CreateJob"); + command["name"] = "testSequential1"; + command["sequentialKey"] = "test_key_finish"; + STable response1 = tester->executeWaitVerifyContentTable(command); + string jobID1 = response1["jobID"]; + + // Create second job + command.clear(); + command.methodLine = "CreateJob"; + command["name"] = "testSequential2"; + command["sequentialKey"] = "test_key_finish"; + STable response2 = tester->executeWaitVerifyContentTable(command); + string jobID2 = response2["jobID"]; + + // Verify second job is WAITING + SQResult result; + clusterTester->getTester(0).readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "WAITING"); + + // Get first job to put it in RUNNING state + command.clear(); + command.methodLine = "GetJob"; + command["name"] = "testSequential1"; + tester->executeWaitVerifyContent(command); + + // Finish first job + command.clear(); + command.methodLine = "FinishJob"; + command["jobID"] = jobID1; + tester->executeWaitVerifyContent(command); + + // Verify first job is deleted + clusterTester->getTester(0).readDB("SELECT * FROM jobs WHERE jobID = " + jobID1 + ";", result); + ASSERT_TRUE(result.empty()); + + // Verify second job is now QUEUED + clusterTester->getTester(0).readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "QUEUED"); + } } __FinishJobTest; diff --git a/test/tests/jobs/CreateJobTest.cpp b/test/tests/jobs/CreateJobTest.cpp index 63be00361..a7e1ab7d7 100644 --- a/test/tests/jobs/CreateJobTest.cpp +++ b/test/tests/jobs/CreateJobTest.cpp @@ -28,6 +28,11 @@ struct CreateJobTest : tpunit::TestFixture TEST(CreateJobTest::retryLifecycle), TEST(CreateJobTest::retryWithChildren), TEST(CreateJobTest::getManualJobWithRetryAfter), + TEST(CreateJobTest::createWithSequentialKey), + TEST(CreateJobTest::sequentialKeyWaitsWhenQueued), + TEST(CreateJobTest::sequentialKeyWaitsWhenRunning), + TEST(CreateJobTest::differentSequentialKeysDontInterfere), + TEST(CreateJobTest::sequentialJobsRunInOrder), AFTER(CreateJobTest::tearDown), AFTER_CLASS(CreateJobTest::tearDownClass)) { @@ -616,4 +621,171 @@ struct CreateJobTest : tpunit::TestFixture command["jobID"] = jobID; tester->executeWaitVerifyContent(command); } + + // First job with sequentialKey should be QUEUED + void createWithSequentialKey() + { + SData command("CreateJob"); + command["name"] = "testSequential"; + command["sequentialKey"] = "test_key_1"; + STable response = tester->executeWaitVerifyContentTable(command); + ASSERT_GREATER_THAN(stol(response["jobID"]), 0); + + SQResult result; + tester->readDB("SELECT state, sequentialKey FROM jobs WHERE jobID = " + response["jobID"] + ";", result); + ASSERT_EQUAL(result.size(), 1); + ASSERT_EQUAL(result[0][0], "QUEUED"); + ASSERT_EQUAL(result[0][1], "test_key_1"); + } + + // Second job with same sequentialKey should be WAITING when first is QUEUED + void sequentialKeyWaitsWhenQueued() + { + // Create first job + SData command("CreateJob"); + command["name"] = "testSequential1"; + command["sequentialKey"] = "test_key_2"; + STable response1 = tester->executeWaitVerifyContentTable(command); + string jobID1 = response1["jobID"]; + + // Create second job with same sequentialKey + command.clear(); + command.methodLine = "CreateJob"; + command["name"] = "testSequential2"; + command["sequentialKey"] = "test_key_2"; + STable response2 = tester->executeWaitVerifyContentTable(command); + string jobID2 = response2["jobID"]; + + // Verify first job is QUEUED + SQResult result; + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID1 + ";", result); + ASSERT_EQUAL(result[0][0], "QUEUED"); + + // Verify second job is WAITING + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "WAITING"); + } + + // Second job with same sequentialKey should be WAITING when first is RUNNING + void sequentialKeyWaitsWhenRunning() + { + // Create first job + SData command("CreateJob"); + command["name"] = "testSequential1"; + command["sequentialKey"] = "test_key_3"; + STable response1 = tester->executeWaitVerifyContentTable(command); + string jobID1 = response1["jobID"]; + + // Get the first job to put it in RUNNING state + command.clear(); + command.methodLine = "GetJob"; + command["name"] = "testSequential1"; + tester->executeWaitVerifyContent(command); + + // Verify first job is RUNNING + SQResult result; + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID1 + ";", result); + ASSERT_EQUAL(result[0][0], "RUNNING"); + + // Create second job with same sequentialKey + command.clear(); + command.methodLine = "CreateJob"; + command["name"] = "testSequential2"; + command["sequentialKey"] = "test_key_3"; + STable response2 = tester->executeWaitVerifyContentTable(command); + string jobID2 = response2["jobID"]; + + // Verify second job is WAITING + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "WAITING"); + } + + // Jobs with different sequentialKeys should not interfere with each other + void differentSequentialKeysDontInterfere() + { + // Create jobs with different sequentialKeys + SData command("CreateJob"); + command["name"] = "testSequentialA"; + command["sequentialKey"] = "key_A"; + STable responseA = tester->executeWaitVerifyContentTable(command); + string jobIDA = responseA["jobID"]; + + command.clear(); + command.methodLine = "CreateJob"; + command["name"] = "testSequentialB"; + command["sequentialKey"] = "key_B"; + STable responseB = tester->executeWaitVerifyContentTable(command); + string jobIDB = responseB["jobID"]; + + // Both should be QUEUED since they have different keys + SQResult result; + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobIDA + ";", result); + ASSERT_EQUAL(result[0][0], "QUEUED"); + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobIDB + ";", result); + ASSERT_EQUAL(result[0][0], "QUEUED"); + } + + // Three jobs with same sequentialKey run in order + void sequentialJobsRunInOrder() + { + // Create three jobs + SData command("CreateJob"); + command["name"] = "testSequential1"; + command["sequentialKey"] = "test_key_order"; + STable response1 = tester->executeWaitVerifyContentTable(command); + string jobID1 = response1["jobID"]; + + command.clear(); + command.methodLine = "CreateJob"; + command["name"] = "testSequential2"; + command["sequentialKey"] = "test_key_order"; + STable response2 = tester->executeWaitVerifyContentTable(command); + string jobID2 = response2["jobID"]; + + command.clear(); + command.methodLine = "CreateJob"; + command["name"] = "testSequential3"; + command["sequentialKey"] = "test_key_order"; + STable response3 = tester->executeWaitVerifyContentTable(command); + string jobID3 = response3["jobID"]; + + // Verify states: first QUEUED, rest WAITING + SQResult result; + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID1 + ";", result); + ASSERT_EQUAL(result[0][0], "QUEUED"); + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "WAITING"); + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID3 + ";", result); + ASSERT_EQUAL(result[0][0], "WAITING"); + + // Get and finish first job + command.clear(); + command.methodLine = "GetJob"; + command["name"] = "testSequential1"; + tester->executeWaitVerifyContent(command); + command.clear(); + command.methodLine = "FinishJob"; + command["jobID"] = jobID1; + tester->executeWaitVerifyContent(command); + + // Verify second job is now QUEUED, third still WAITING + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "QUEUED"); + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID3 + ";", result); + ASSERT_EQUAL(result[0][0], "WAITING"); + + // Get and finish second job + command.clear(); + command.methodLine = "GetJob"; + command["name"] = "testSequential2"; + tester->executeWaitVerifyContent(command); + command.clear(); + command.methodLine = "FinishJob"; + command["jobID"] = jobID2; + tester->executeWaitVerifyContent(command); + + // Verify third job is now QUEUED + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID3 + ";", result); + ASSERT_EQUAL(result[0][0], "QUEUED"); + } } __CreateJobTest; diff --git a/test/tests/jobs/DeleteJobTest.cpp b/test/tests/jobs/DeleteJobTest.cpp index 146e0dac8..c055cadbf 100644 --- a/test/tests/jobs/DeleteJobTest.cpp +++ b/test/tests/jobs/DeleteJobTest.cpp @@ -11,6 +11,7 @@ struct DeleteJobTest : tpunit::TestFixture TEST(DeleteJobTest::deleteJobWithChild), TEST(DeleteJobTest::deleteRunningJob), TEST(DeleteJobTest::deleteFinishedJob), + TEST(DeleteJobTest::deleteJobPromotesWaitingJob), AFTER(DeleteJobTest::tearDown), AFTER_CLASS(DeleteJobTest::tearDownClass)) { @@ -197,4 +198,42 @@ struct DeleteJobTest : tpunit::TestFixture command["jobID"] = parentID; tester->executeWaitVerifyContent(command); } + + // DeleteJob should promote the next WAITING job with same sequentialKey + void deleteJobPromotesWaitingJob() + { + // Create first job + SData command("CreateJob"); + command["name"] = "testSequential1"; + command["sequentialKey"] = "test_key_delete"; + STable response1 = tester->executeWaitVerifyContentTable(command); + string jobID1 = response1["jobID"]; + + // Create second job + command.clear(); + command.methodLine = "CreateJob"; + command["name"] = "testSequential2"; + command["sequentialKey"] = "test_key_delete"; + STable response2 = tester->executeWaitVerifyContentTable(command); + string jobID2 = response2["jobID"]; + + // Verify second job is WAITING + SQResult result; + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "WAITING"); + + // Delete first job + command.clear(); + command.methodLine = "DeleteJob"; + command["jobID"] = jobID1; + tester->executeWaitVerifyContent(command); + + // Verify first job is deleted + tester->readDB("SELECT * FROM jobs WHERE jobID = " + jobID1 + ";", result); + ASSERT_TRUE(result.empty()); + + // Verify second job is now QUEUED + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "QUEUED"); + } } __DeleteJobTest; diff --git a/test/tests/jobs/FailJobTest.cpp b/test/tests/jobs/FailJobTest.cpp index b29c0e57f..fbc84761e 100644 --- a/test/tests/jobs/FailJobTest.cpp +++ b/test/tests/jobs/FailJobTest.cpp @@ -12,6 +12,7 @@ struct FailJobTest : tpunit::TestFixture TEST(FailJobTest::notInRunningRunqueuedState), TEST(FailJobTest::failJobInRunningState), TEST(FailJobTest::failJobInRunqueuedState), + TEST(FailJobTest::failJobPromotesWaitingJob), AFTER(FailJobTest::tearDown), AFTER_CLASS(FailJobTest::tearDownClass)) { @@ -123,4 +124,48 @@ struct FailJobTest : tpunit::TestFixture tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID + ";", result); ASSERT_EQUAL(result[0][0], "FAILED"); } + + // FailJob should promote the next WAITING job with same sequentialKey + void failJobPromotesWaitingJob() + { + // Create first job + SData command("CreateJob"); + command["name"] = "testSequential1"; + command["sequentialKey"] = "test_key_fail"; + STable response1 = tester->executeWaitVerifyContentTable(command); + string jobID1 = response1["jobID"]; + + // Create second job + command.clear(); + command.methodLine = "CreateJob"; + command["name"] = "testSequential2"; + command["sequentialKey"] = "test_key_fail"; + STable response2 = tester->executeWaitVerifyContentTable(command); + string jobID2 = response2["jobID"]; + + // Verify second job is WAITING + SQResult result; + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "WAITING"); + + // Get first job to put it in RUNNING state + command.clear(); + command.methodLine = "GetJob"; + command["name"] = "testSequential1"; + tester->executeWaitVerifyContent(command); + + // Fail first job + command.clear(); + command.methodLine = "FailJob"; + command["jobID"] = jobID1; + tester->executeWaitVerifyContent(command); + + // Verify first job is FAILED + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID1 + ";", result); + ASSERT_EQUAL(result[0][0], "FAILED"); + + // Verify second job is now QUEUED + tester->readDB("SELECT state FROM jobs WHERE jobID = " + jobID2 + ";", result); + ASSERT_EQUAL(result[0][0], "QUEUED"); + } } __FailJobTest;