Compress cache via brotli

bin/data

@@ -36,7 +36,7 @@ const command = clap.command('data', '[config] [model]')
             pretty
         };
 
-        getData(options).then(({ stream, size } = {}) => {
+        getData(options).then(({ stream, size, brotli } = {}) => {
             if (outputFile) {
                 stream.pipe(fs.createWriteStream(outputFile));
                 return;
@@ -50,7 +50,7 @@ const command = clap.command('data', '[config] [model]')
                     return;
                 }
 
-                process.send({ payload: 'stream', size });
+                process.send({ payload: 'stream', size, brotli });
                 stream.pipe(fs.createWriteStream(null, { fd: 4 }));
                 return;
             }

lib/data.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
-const { stringifyInfo: jsonInfo, stringifyStream: createJsonStringifyStream } = require('@discoveryjs/json-ext');
+const zlib = require('zlib');
+const { stringifyStream: createJsonStringifyStream } = require('@discoveryjs/json-ext');
 const bootstrap = require('./shared/bootstrap');
 const getCacheFilename = require('./shared/get-cache-filename');
 const ensureDir = require('./shared/ensure-dir');
@@ -43,7 +44,8 @@ function getData(modelConfig, { rewriteCache, warmup, pretty } = {}) {
                 // try to read from a cache
                 return Promise.resolve({
                     stream: fs.createReadStream(cacheFile),
-                    size: stat.size
+                    size: stat.size,
+                    brotli: true
                 });
             }
         } catch (e) {}
@@ -55,6 +57,7 @@ function getData(modelConfig, { rewriteCache, warmup, pretty } = {}) {
         if (cacheFile) {
             const startWriteTime = Date.now();
             const writeToFileStream = jsonStream
+                .pipe(zlib.createBrotliCompress())
                 .pipe(fs.createWriteStream(ensureDir(cacheFile)))
                 .on('finish', () => {
                     console.log('[cache]', modelConfig.slug, 'cache written in', Date.now() - startWriteTime, 'ms');
@@ -70,15 +73,7 @@ function getData(modelConfig, { rewriteCache, warmup, pretty } = {}) {
             }
         }
 
-        const result = { stream: jsonStream };
-
-        const dataInfo = jsonInfo(data);
-
-        if (!dataInfo.async.length && !dataInfo.circular.length) {
-            result.size = dataInfo.minLength;
-        }
-
-        return result;
+        return { stream: jsonStream };
     });
 }
 

lib/serve.js

@@ -148,9 +148,13 @@ function generateDataJson(modelConfig, options) {
         }
 
         return request
-            .then(({ stream, size }) => {
+            .then(({ stream, size, brotli }) => {
                 res.set('Content-Type', 'application/json');
 
+                if (brotli) {
+                    res.set('Content-Encoding', 'br');
+                }
+
                 if (size) {
                     res.set('Content-Length', size);
                 }

lib/shared/utils.js

@@ -107,7 +107,8 @@ function runScript(command, args) {
                     if (message.payload === 'stream') {
                         resolve({
                             stream: child.stdio[4],
-                            size: message.size
+                            size: message.size,
+                            brotli: message.brotli
                         });
                     }
                 } catch (e) {}