Merge pull request #16 from emostar/ihavener/issue-14/reject-requests…

…-cleanly-during-exit Ihavener/issue 14/reject requests cleanly during exit
emostar · Oct 14, 2019 · db219c4 · db219c4
2 parents 6eba1de + 7abda34
commit db219c4
Show file tree

Hide file tree

Showing 2 changed files with 78 additions and 34 deletions.
diff --git a/README.md b/README.md
@@ -1,8 +1,8 @@
 # express-graceful-exit
 
-A component in having zero downtime deploys for Node.js with [Express](http://expressjs.com/). It was developed for Express 3.X, so it may need work to be compatible with Express 2.X
+Gracefully decline new requests while shutting down your application. A component that helps support zero downtime deploys for Node.js with [Express](http://expressjs.com/).
 
-This module was originally developed for [Frafty](https://www.frafty.com/), a Daily Fantasy Sports site.
+The project was originally developed for Express v3.X, but is used in production with Express v4.X. Please write up an issue or submit a PR if you find bugs using express-graceful-exit with Express v4.X and higher.
 
 ## Installation
 
@@ -13,11 +13,15 @@ $ npm install express-graceful-exit
 
 ## Compatibility
 
-v0.X.X versions are backwards API compatible, with the caveate that process exit is called in a `setTimeout` block from v0.2.0 forward, so the timing is slightly different between v0.1.0 to v0.2.x+.
+v0.X.X versions are backwards API compatible, with these minor behavior changes:
+1. Process exit is called in a `setTimeout` block from v0.2.0 forward, so the timing is slightly different between v0.1.0 to v0.2.x+.
+2. After exit was triggered, incoming requests were mismanaged prior to v0.5.0. <br> As of v0.5.0 incoming requests are dropped cleanly by default, with new options such as responding with a custom error and/or performing one last request per connection.
 
 ## Usage
 
-The following two components must both be used to enable fully graceful exits.
+The following two components must both be used to enable clean server shutdown, where incoming requests are gracefully declined.
+
+There are multiple exit options for how in-flight requests are handled, ranging from forced exist after a specified deadline to waiting indefinitely for processing to complete.
 
 ### middleware
 
@@ -28,6 +32,9 @@ var express = require('express');
 var app = express();
 var gracefulExit = require('express-graceful-exit');
 
+var server = app.listen(port)
+
+gracefulExit.init(server) // use init() if configured to exit the process after timeout
 app.use(gracefulExit.middleware(app));
 ````
 
@@ -40,7 +47,7 @@ This function tells express to accept no new requests and gracefully closes the
 process.on('message', function(message) {
   if (message === 'shutdown') {
     gracefulExit.gracefulExitHandler(app, server, {
-        socketio: app.settings.socketio
+        <see options below>
     });
   }
 });
@@ -61,30 +68,32 @@ The following options are available:
  __log__             |  Print status messages and errors to the logger  |  false
  __logger__          |  Function that accepts a string to output a log message  |  console.log
  __callback__        |  Optional function that is called with the exit status code once express has shutdown, gracefully or not <br> Use in conjunction with  `exitProcess: false` when the caller handles process shutdown  |  no-op
- __exitProcess__     |  If true, the module calls `process.exit()` when express has shutdown, gracefully or not  |  true
- __exitDelay__       |  Wait timer duration in the final internal callback (triggered either by gracefulExitHandler or the suicideTimeout) if `exitProcess: true`  |  10ms
- __suicideTimeout__  |  How long to wait before giving up on graceful  shutdown, then returns exit code of 1  |  2m 10s (130s)
- __socketio__        |  An instance of `socket.io`, used to close all  open connections after timeout  |  none
+ __performLastRequest__ |  Process the first request received per connection after exit starts, and include a connection close header for callers and load balancers. <br> `false` is the existing behavior, deprecated as of v0.5.0 |  false
+ __errorDuringExit__ |  Respond to incoming requests with an error instead of silently dropping them. <br> `false` is the existing behavior, deprecated as of v0.5.0  |  false
+ __getRejectionError__  |  Function returning rejection error for incoming requests during graceful exit | `function () { return new Error('Server unavailable, no new requests accepted during shutdown') }`
+ __exitProcess__      |  If true, the module calls `process.exit()` when express has shutdown, gracefully or not  |  true
+ __exitDelay__       |  Wait timer duration in the final internal callback (triggered either by gracefulExitHandler or the hard exit handler) if `exitProcess: true`  |  10ms
+  __suicideTimeout__ |  How long to wait before giving up on graceful shutdown, then returns exit code of 1  |  2m 10s (130s)
+ __socketio__        |  An instance of `socket.io`, used to close all open connections after timeout  |  none
  __force__           |  Instructs the module to forcibly close sockets once the suicide timeout elapses. <br> For this option to work you must call `gracefulExit.init(server)` when initializing the HTTP server  |  false
 
 ## Details
 
-To gracefully exit this module will do the following things:
+To gracefully exit this module does the following things:
 
-1. Close the http server so no new connections are accepted
-2. Mark that the server will gracefully exit, so if a connection that is using the Keep-Alive header is still active, it will be told to close the connection  
-The HTTP status code of 502 is returned, so nginx, ELB, etc will try again with a working server
-3. If a socket.io instance is passed in the options, it enumerates all connected clients and disconnects them  
-The client should have code to reconnect on disconnect
-4. Server fully disconnects or the hard exit timer runs
-    1. Once all connected clients are disconnected, the exit handler returns `0`
-    2. OR If there are any remaining connections after `suicideTimeout` ms, the handler returns `1`
-5. In either case, if exitProcess is set to true the exit handler waits exitDelay ms and calls `process.exit`
+1. Closes the http server so no new connections are accepted
+2. Sets connection close header for Keep-Alive connections, if configured for responses</br> The HTTP status code of 502 is returned, so nginx, ELB, etc will try with an active server</br> If `errorDuringExit` and/or `performLastRequest` are set to true, a response is sent with a `Connection: close` header
+3. If a socket.io instance is passed in the options, all connected clients are immediately disconnected (socket.io v0.X through v1.4.x support)</br> The client should have code to reconnect on disconnect
+4. Once the server fully disconnects or the hard exit timer runs
+    1. If all in-flight requests have resolved and/or disconnected, the exit handler returns `0`
+    2. OR if any connections remain after `suicideTimeout` ms, the handler returns `1`
+5. In either case, if exitProcess is set to true the hard exit handler waits exitDelay ms and calls `process.exit(x)`, this allows the logger time to flush and the app's callback to complete, if any
 
 ## Zero Downtime Deploys
 
 This module does not give you zero downtime deploys on its own. It enables the http server to exit gracefully, which when used with a module like naught can provide zero downtime deploys.
 
 #### Author: [Jon Keating](http://twitter.com/emostar)
+This module was originally developed for Frafty (formerly www.frafty.com), a Daily Fantasy Sports site.
 #### Maintainer: [Ivo Havener](https://github.com/ivolucien)
 
diff --git a/lib/graceful-exit.js b/lib/graceful-exit.js
@@ -1,11 +1,24 @@
 
 var _ = require('underscore');
+var inspect = require('util').inspect;
 
 var sockets = [];
 var options = {};
 var hardExitTimer;
 var connectionsClosed = false;
 
+var defaultOptions = {
+  errorDuringExit   : false, // false is existing behavior, deprecated as of v0.5.0
+  performLastRequest: false, // false is existing behavior, deprecated as of v0.5.0
+  log               : false,
+  logger            : console.log,
+  getRejectionError : function (err) { return err; },
+  suicideTimeout    : 2*60*1000 + 10*1000,  // 2m10s (nodejs default is 2m)
+  exitProcess       : true,
+  exitDelay         : 10,    // wait in ms before process.exit, if exitProcess true
+  force             : false
+};
+
 function logger (str) {
   if (options.log) {
     options.logger(str);
@@ -90,15 +103,6 @@ exports.hardExitHandler = function hardExitHandler () {
   hardExitTimer = null;
 };
 
-var defaultOptions = {
-  log               : false,
-  logger            : console.log,
-  suicideTimeout    : 2*60*1000 + 10*1000,  // 2m10s (nodejs default is 2m)
-  exitProcess       : true,
-  exitDelay         : 10,    // wait in ms before process.exit, if exitProcess true
-  force             : false
-};
-
 exports.gracefulExitHandler = function gracefulExitHandler (app, server, _options) {
   // Get the options set up
   if (!_options) {
@@ -140,18 +144,49 @@ exports.gracefulExitHandler = function gracefulExitHandler (app, server, _option
   hardExitTimer = setTimeout(exports.hardExitHandler, options.suicideTimeout);
 };
 
+exports.handleFinalRequests = function handleFinalRequests (req, res, next) {
+  var headers = inspect(req.headers) || '?'; // safe object to string
+
+  if (options.performLastRequest && connection.lastRequestStarted === false) {
+    logger('Server exiting, performing last request for this connection. Headers: ' + headers);
+
+    req.connection.lastRequestStarted = true;
+    return next();
+  }
+
+  if (options.errorDuringExit) {
+    logger('Server unavailable, incoming request rejected with error. Headers: ' + headers);
+
+    return next(
+      options.getRejectionError() ||
+      defaultOptions.getRejectionError(
+        new Error('Server unavailable, no new requests accepted during shutdown')
+      )
+    );
+  }
+
+  // else silently drop request without response (existing deprecated behavior)
+  logger('Server unavailable, incoming request dropped silently. Headers: ' + headers);
+
+  res.end(); // end request without calling next()
+  return null;
+};
+
 exports.middleware = function middleware (app) {
   // This flag is used to signal the below middleware when the server wants to stop.
-  // New connections are handled for us by Node, but existing connections using the
-  // Keep-Alive header require this workaround to close.
   app.set('graceful_exit', false);
 
   return function checkIfExitingGracefully (req, res, next) {
-    if (app.settings.graceful_exit === true) {
-      // sorry keep-alive connections, but we need to part ways
-      req.connection.setTimeout(1);
+    var connection = req.connection || {};
+
+    if (app.settings.graceful_exit === false) {
+      connection.lastRequestStarted = connection.lastRequestStarted || false;
+      next();
     }
 
-    next();
+    // Set connection closing header for response, if any. Fix to issue 14, thank you HH
+    res.set('Connection', 'close');
+
+    return exports.handleFinalRequests(req, res, next);
   };
 };