From c05f59c05ea752a748680849e31f940e21230d1d Mon Sep 17 00:00:00 2001
From: James McKinney <26463+jpmckinney@users.noreply.github.com>
Date: Fri, 12 Jul 2024 16:43:30 -0400
Subject: [PATCH] fix: Fix #327 when managing multiple projects

---
 docs/news.rst         |  5 +++++
 scrapyd/interfaces.py |  3 +++
 scrapyd/poller.py     | 12 +++++++++---
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 24276dbb..f4bd1625 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -19,6 +19,11 @@ Changed
 
 - Add a confirmation dialog to the Cancel button.
 
+Fixed
+~~~~~
+
+- When managing multiple projects, the next pending job for all but one project was unreported by the :ref:`daemonstatus.json` and :ref:`listjobs.json` webservices, and was not cancellable by the :ref:`cancel.json` webservice.
+
 1.4.3 (2023-09-25)
 ------------------
 
diff --git a/scrapyd/interfaces.py b/scrapyd/interfaces.py
index 7c9d1c8a..18fcccc1 100644
--- a/scrapyd/interfaces.py
+++ b/scrapyd/interfaces.py
@@ -46,6 +46,9 @@ def next():
         -  a unique identifier for this run in the `_job` key
 
         This message will be passed later to IEnvironment.get_environment().
+
+        Called ``max_proc`` times when the launcher starts, and each time a
+        Scrapy process ends.
         """
 
     def update_projects():
diff --git a/scrapyd/poller.py b/scrapyd/poller.py
index 28cbe0ff..b54041ec 100644
--- a/scrapyd/poller.py
+++ b/scrapyd/poller.py
@@ -15,16 +15,22 @@ def __init__(self, config):
 
     @inlineCallbacks
     def poll(self):
-        if not self.dq.waiting:
-            return
         for project, queue in self.queues.items():
+            # If the "waiting" backlog is empty (that is, if the maximum number of Scrapy processes are running):
+            if not self.dq.waiting:
+                return
             count = yield maybeDeferred(queue.count)
             if count:
                 message = yield maybeDeferred(queue.pop)
-                if message is not None:  # In case of a concurrently accessed queue
+                # The message can be None if, for example, two Scrapyd instances share a spider queue database.
+                if message is not None:
+                    # Pop a dummy item from the "waiting" backlog. and fire the message.
                     returnValue(self.dq.put(self._message(message, project)))
 
     def next(self):
+        """
+        Add a dummy item to the "waiting" backlog (based on Twisted's implementation of DeferredQueue).
+        """
         return self.dq.get()
 
     def update_projects(self):