fix: run-workers --by suite parallelization broken by test file sorting (#5412)

The sorting of test files in loadTests() (added in #5386) broke the
--by suite parallelization. When files were sorted before worker
distribution, all workers could receive the same tests instead of
different suites being distributed to different workers.

Fix: Move testFiles.sort() from loadTests() to run(). This ensures:
- Worker distribution uses original (unsorted) file order for consistent
  distribution across workers
- Test execution still uses alphabetical order (sorted in run())

Added unit test to verify files are not sorted after loadTests().

Fixes #5412

Co-Authored-By: Claude Opus 4.5 <[email protected]>

Author: mirao

lib/codecept.js

@@ -284,6 +284,9 @@ class Codecept {
       // Ignore if gherkin module not available
     }
 
+    // Sort test files alphabetically for consistent execution order
+    this.testFiles.sort()
+
     return new Promise((resolve, reject) => {
       const mocha = container.mocha()
       mocha.files = this.testFiles

test/unit/alphabetical_order_test.js

@@ -0,0 +1,107 @@
+import { expect } from 'chai'
+import Codecept from '../../lib/codecept.js'
+import Container from '../../lib/container.js'
+import path from 'path'
+import fs from 'fs'
+import { fileURLToPath } from 'url'
+import { dirname } from 'path'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
+
+describe('Test Files Alphabetical Order', () => {
+  let codecept
+  const tempDir = path.join(__dirname, '../data/sandbox/configs/alphabetical_order')
+  const config = {
+    tests: `${tempDir}/*_test.js`,
+    gherkin: { features: null },
+    output: './output',
+    hooks: [],
+  }
+
+  before(() => {
+    if (!fs.existsSync(tempDir)) {
+      fs.mkdirSync(tempDir, { recursive: true })
+    }
+
+    fs.writeFileSync(
+      path.join(tempDir, 'zzz_test.js'),
+      `
+        Feature('Test');
+        Scenario('zzz test', () => {});
+      `,
+    )
+    fs.writeFileSync(
+      path.join(tempDir, 'aaa_test.js'),
+      `
+        Feature('Test');
+        Scenario('aaa test', () => {});
+      `,
+    )
+    fs.writeFileSync(
+      path.join(tempDir, 'mmm_test.js'),
+      `
+        Feature('Test');
+        Scenario('mmm test', () => {});
+      `,
+    )
+  })
+
+  after(() => {
+    const files = ['zzz_test.js', 'aaa_test.js', 'mmm_test.js']
+    files.forEach(file => {
+      const filePath = path.join(tempDir, file)
+      if (fs.existsSync(filePath)) {
+        fs.unlinkSync(filePath)
+      }
+    })
+  })
+
+  beforeEach(async () => {
+    codecept = new Codecept(config, {})
+    await codecept.init(path.join(__dirname, '../data/sandbox'))
+  })
+
+  afterEach(() => {
+    Container.clear()
+  })
+
+  it('should sort test files alphabetically when run() is called', async () => {
+    codecept.loadTests()
+
+    if (codecept.testFiles.length === 0) {
+      console.log('No test files found, skipping test')
+      return
+    }
+
+    // Capture the files that would be passed to mocha during run()
+    let filesPassedToMocha = null
+    const mocha = Container.mocha()
+    mocha.run = callback => {
+      filesPassedToMocha = [...mocha.files]
+      // Call callback immediately to complete the run
+      if (callback) callback()
+    }
+
+    // Call run() which should sort files before passing to mocha
+    await codecept.run()
+
+    // Verify files passed to mocha are sorted alphabetically
+    expect(filesPassedToMocha).to.not.be.null
+    const filenames = filesPassedToMocha.map(filePath => path.basename(filePath))
+    const sortedFilenames = [...filenames].sort()
+
+    expect(filenames).to.deep.equal(sortedFilenames, 'Files should be sorted alphabetically for execution')
+
+    const aaaIndex = filenames.findIndex(f => f.includes('aaa_test.js'))
+    const mmmIndex = filenames.findIndex(f => f.includes('mmm_test.js'))
+    const zzzIndex = filenames.findIndex(f => f.includes('zzz_test.js'))
+
+    expect(aaaIndex).to.be.greaterThan(-1)
+    expect(mmmIndex).to.be.greaterThan(-1)
+    expect(zzzIndex).to.be.greaterThan(-1)
+
+    expect(aaaIndex).to.be.lessThan(mmmIndex, 'aaa_test.js should come before mmm_test.js')
+    expect(mmmIndex).to.be.lessThan(zzzIndex, 'mmm_test.js should come before zzz_test.js')
+  })
+})

test/unit/worker_test.js

@@ -382,4 +382,34 @@ describe('Workers', function () {
 
     workers.run()
   })
+
+  it('should distribute suites across workers for --by suite (issue #5412)', async () => {
+    // This test verifies the fix for issue #5412:
+    // Test files should NOT be sorted in loadTests() because that affects worker distribution.
+    // Sorting should only happen in run() for execution order.
+    //
+    // The bug was: sorting in loadTests() changed the order of suites during distribution,
+    // causing all workers to receive the same tests instead of different suites.
+
+    // Ensure clean state
+    Container.clear()
+    Container.createMocha()
+
+    const workerConfig = {
+      by: 'suite',
+      testConfig: './test/data/sandbox/codecept.customworker.js',
+    }
+
+    const workers = new Workers(3, workerConfig)
+    await workers._ensureInitialized()
+
+    // Verify that test files were loaded
+    const testFiles = workers.codecept.testFiles
+    expect(testFiles.length).to.be.greaterThan(0, 'Test files should be loaded')
+
+    // Verify that suites are distributed across multiple worker groups
+    const groups = workers.testGroups
+    const nonEmptyGroups = groups.filter(g => g.length > 0)
+    expect(nonEmptyGroups.length).to.be.greaterThan(1, 'Suites should be distributed across multiple worker groups')
+  })
 })