index.js

'use strict';

const fs = require('fs');
const path = require('path');
const { spawn } = require('child_process');
const crypto = require('crypto');

let tmp;
let archives_disabled = false;

exports.re = {
  ct: /^([^/]+\/[^;\r\n ]+)/, // content type
};

exports.register = function () {
  this.load_tmp_module();
  this.load_attachment_ini();

  this.load_n_compile_re('file', 'attachment.filename.regex');
  this.load_n_compile_re('ctype', 'attachment.ctype.regex');
  this.load_n_compile_re('archive', 'attachment.archive.filename.regex');

  this.register_hook('data_post', 'wait_for_attachment_hooks');
  this.register_hook('data_post', 'check_attachments');
};

exports.load_tmp_module = function () {
  try {
    tmp = require('tmp');
    tmp.setGracefulCleanup();
  } catch (ignore) {
    archives_disabled = true;
    this.logwarn(
      `the 'tmp' module is required to extract filenames from archives`,
    );
  }
};

exports.load_attachment_ini = function () {
  const plugin = this;

  plugin.cfg = plugin.config.get('attachment.ini', () => {
    plugin.load_attachment_ini();
  });

  plugin.cfg.timeout = (plugin.cfg.main.timeout || 30) * 1000;

  // repair a mismatch between legacy docs and code
  const extns =
    plugin.cfg.archive && plugin.cfg.archive.extensions
      ? plugin.cfg.archive.extensions // new
      : plugin.cfg.main.archive_extensions // old code
        ? plugin.cfg.main.archive_extensions
        : plugin.cfg.main.archive_extns // old docs
          ? plugin.cfg.main.archive_extns
          : 'zip tar tgz taz z gz rar 7z';

  plugin.cfg.archive.exts = this.options_to_object(extns);

  plugin.cfg.archive.max_depth =
    plugin.cfg.archive && plugin.cfg.archive.max_depth
      ? plugin.cfg.archive.max_depth // new
      : plugin.cfg.main.archive_max_depth // old
        ? plugin.cfg.main.archive_max_depth
        : 5;

  plugin.load_dissallowed_extns();
};

exports.find_bsdtar_path = (cb) => {
  let found = false;
  let i = 0;
  ['/bin', '/usr/bin', '/usr/local/bin'].forEach((dir) => {
    if (found) return;
    i++;
    fs.stat(`${dir}/bsdtar`, (err) => {
      i--;
      if (found) return;
      if (err) {
        if (i === 0) cb(new Error('bsdtar not found'));
        return;
      }
      found = true;
      cb(null, dir);
    });
    if (i === 0) cb(new Error('bsdtar not found'));
  });
};

exports.hook_init_master = exports.hook_init_child = function (next) {
  const plugin = this;

  plugin.find_bsdtar_path((err, dir) => {
    if (err) {
      archives_disabled = true;
      plugin.logwarn(
        `This plugin requires the 'bsdtar' binary to extract filenames from archive files`,
      );
    } else {
      plugin.logdebug(`found bsdtar in ${dir}`);
      plugin.bsdtar_path = `${dir}/bsdtar`;
    }
    next();
  });
};

exports.load_dissallowed_extns = function () {
  const plugin = this;

  if (!plugin.cfg.main.disallowed_extensions) return;

  if (!plugin.re) plugin.re = {};
  plugin.re.bad_extn = new RegExp(
    '\\.(?:' +
      plugin.cfg.main.disallowed_extensions
        .replace(/\s+/, ' ')
        .split(/[;, ]/)
        .join('|') +
      ')$',
    'i',
  );
};

exports.load_n_compile_re = function (name, file) {
  const plugin = this;
  const valid_re = [];

  const try_re = plugin.config.get(file, 'list', function () {
    plugin.load_n_compile_re(name, file);
  });

  for (let r = 0; r < try_re.length; r++) {
    try {
      const reg = new RegExp(try_re[r], 'i');
      valid_re.push(reg);
    } catch (e) {
      this.logerror(`skipping invalid regexp: /${try_re[r]}/ (${e})`);
    }
  }

  if (!plugin.re) plugin.re = {};
  plugin.re[name] = valid_re;
};

exports.options_to_object = function (options) {
  if (!options) return false;

  const res = {};
  options
    .toLowerCase()
    .replace(/\s+/, ' ')
    .split(/[;, ]/)
    .forEach((opt) => {
      if (!opt) return;
      res[opt.trim()] = true;
    });

  if (Object.keys(res).length) return res;
  return false;
};

exports.unarchive_recursive = async function (
  connection,
  f,
  archive_file_name,
  cb,
) {
  if (archives_disabled) {
    connection.logdebug(this, 'archive support disabled');
    return cb();
  }

  const plugin = this;
  const tmpfiles = [];

  let timeouted = false;
  let encrypted = false;
  let depthExceeded = false;

  function timeoutedSpawn(cmd_path, args, env, pipe_stdout_ws) {
    connection.logdebug(plugin, `running "${cmd_path} ${args.join(' ')}"`);

    return new Promise(function (resolve, reject) {
      let output = '';
      const p = spawn(cmd_path, args, env);

      // Start timer
      let timeout = false;
      const timer = setTimeout(() => {
        timeout = timeouted = true;
        p.kill();

        reject(`command "${cmd_path} ${args}" timed out`);
      }, plugin.cfg.timeout);

      if (pipe_stdout_ws) {
        p.stdout.pipe(pipe_stdout_ws);
      } else {
        p.stdout.on('data', (data) => (output += data));
      }

      p.stderr.on('data', (data) => {
        if (data.includes('Incorrect passphrase')) {
          encrypted = true;
        }

        // it seems that stderr might be sometimes filled after exit so we rather print it out than wait for result
        connection.logdebug(plugin, `"${cmd_path} ${args.join(' ')}": ${data}`);
      });

      p.on('exit', (code, signal) => {
        if (timeout) return;
        clearTimeout(timer);

        if (code && code > 0) {
          // Error was returned
          return reject(
            `"${cmd_path} ${args.join(' ')}" returned error code: ${code}}`,
          );
        }

        if (signal) {
          // Process terminated due to signal
          return reject(
            `"${cmd_path} ${args.join(' ')}" terminated by signal: ${signal}`,
          );
        }

        resolve(output);
      });
    });
  }

  function createTmp() {
    // might be better to use async version of tmp in future not cb based
    return new Promise((resolve, reject) => {
      tmp.file((err, tmpfile, fd) => {
        if (err) reject(err);

        const t = {};
        t.name = tmpfile;
        t.fd = fd;

        resolve(t);
      });
    });
  }

  async function unpackArchive(in_file, file) {
    const t = await createTmp();
    tmpfiles.push([t.fd, t.name]);

    connection.logdebug(
      plugin,
      `created tmp file: ${t.name} (fd=${t.fd}) for file ${file}`,
    );

    const tws = fs.createWriteStream(t.name);
    try {
      // bsdtar seems to be asking for password if archive is encrypted workaround with --passphrase will end up
      // with "Incorrect passphrase" for encrypted archives, but will be ignored with nonencrypted
      await timeoutedSpawn(
        plugin.bsdtar_path,
        [
          '-Oxf',
          in_file,
          `--include=${file}`,
          '--passphrase',
          'deliberately_invalid',
        ],
        {
          cwd: '/tmp',
          env: {
            LANG: 'C',
          },
        },
        tws,
      );
    } catch (e) {
      connection.logdebug(plugin, e);
    }
    return t;
  }

  async function listArchive(in_file) {
    try {
      const lines = await timeoutedSpawn(
        plugin.bsdtar_path,
        ['-tf', in_file, '--passphrase', 'deliberately_invalid'],
        {
          cwd: '/tmp',
          env: { LANG: 'C' },
        },
      );

      // Extract non-empty filenames
      return lines.split(/\r?\n/).filter((fl) => fl);
    } catch (e) {
      connection.logdebug(plugin, e);
      return [];
    }
  }

  function deleteTempFiles() {
    tmpfiles.forEach((t) => {
      fs.close(t[0], () => {
        connection.logdebug(plugin, `closed fd: ${t[0]}`);
        fs.unlink(t[1], () => {
          connection.logdebug(plugin, `deleted tempfile: ${t[1]}`);
        });
      });
    });
  }

  async function processFile(in_file, prefix, file, depth) {
    let result = [(prefix ? `${prefix}/` : '') + file];

    connection.logdebug(
      plugin,
      `found file: ${prefix ? `${prefix}/` : ''}${file} depth=${depth}`,
    );

    if (!plugin.isArchive(path.extname(file.toLowerCase()))) {
      return result;
    }

    connection.logdebug(
      plugin,
      `need to extract file: ${prefix ? `${prefix}/` : ''}${file}`,
    );

    const t = await unpackArchive(in_file, file);

    // Recurse
    try {
      result = result.concat(
        await listFiles(t.name, (prefix ? `${prefix}/` : '') + file, depth + 1),
      );
    } catch (e) {
      connection.logdebug(plugin, e);
    }

    return result;
  }

  async function listFiles(in_file, prefix, depth) {
    const result = [];
    depth = depth || 0;

    if (timeouted) {
      connection.logdebug(
        plugin,
        `already timeouted, not going to process ${prefix ? `${prefix}/` : ''}${in_file}`,
      );
      return result;
    }

    if (depth >= plugin.cfg.archive.max_depth) {
      depthExceeded = true;
      connection.logdebug(
        plugin,
        `hit maximum depth with ${prefix ? `${prefix}/` : ''}${in_file}`,
      );
      return result;
    }

    const fls = await listArchive(in_file);
    await Promise.all(
      fls.map(async (file) => {
        const output = await processFile(in_file, prefix, file, depth + 1);
        result.push(...output);
      }),
    );

    connection.loginfo(
      plugin,
      `finish (${prefix ? `${prefix}/` : ''}${in_file}): count=${result.length} depth=${depth}`,
    );
    return result;
  }

  setTimeout(() => {
    timeouted = true;
  }, plugin.cfg.timeout);

  const files = await listFiles(f, archive_file_name);
  deleteTempFiles();

  if (timeouted) {
    cb(new Error('archive extraction timeouted'), files);
  } else if (depthExceeded) {
    cb(new Error('maximum archive depth exceeded'), files);
  } else if (encrypted) {
    cb(new Error('archive encrypted'), files);
  } else {
    cb(null, files);
  }
};

exports.compute_and_log_md5sum = function (
  connection,
  ctype,
  filename,
  stream,
) {
  const plugin = this;
  const md5 = crypto.createHash('md5');
  let bytes = 0;

  stream.on('data', (data) => {
    md5.update(data);
    bytes += data.length;
  });

  stream.once('end', () => {
    stream.pause();

    const digest = md5.digest('hex') || '';
    const ct = plugin.content_type(connection, ctype);

    connection.transaction.notes.attachments.push({
      ctype: ct,
      filename,
      extension: plugin.file_extension(filename),
      md5: digest,
    });

    connection.transaction.results.push(plugin, {
      attach: {
        file: filename,
        ctype: ct,
        md5: digest,
        bytes,
      },
      emit: true,
    });
    connection.loginfo(
      plugin,
      `file="${filename}" ctype="${ctype}" md5=${digest} bytes=${bytes}`,
    );
  });
};

exports.file_extension = function (filename) {
  if (!filename) return '';

  const ext_match = filename.match(/\.([^. ]+)$/);
  if (!ext_match || !ext_match[1]) return '';

  return ext_match[1].toLowerCase();
};

exports.content_type = function (connection, ctype) {
  const plugin = this;

  const ct_match = ctype.match(plugin.re.ct);
  if (!ct_match || !ct_match[1]) return 'unknown/unknown';

  connection.logdebug(plugin, `found content type: ${ct_match[1]}`);
  connection.transaction.notes.attachment_ctypes.push(ct_match[1]);
  return ct_match[1].toLowerCase();
};

exports.isArchive = function (file_ext) {
  // check with and without the dot prefixed
  if (this.cfg.archive.exts[file_ext]) return true;
  if (file_ext[0] === '.' && this.cfg.archive.exts[file_ext.substring(1)])
    return true;
  return false;
};

exports.start_attachment = function (
  connection,
  ctype,
  filename,
  body,
  stream,
) {
  const plugin = this;
  const txn = connection?.transaction;

  function next() {
    if (txn?.notes?.attachment_next && txn.notes.attachment_count === 0) {
      return txn.notes.attachment_next();
    }
  }

  let file_ext = '.unknown';

  if (filename) {
    file_ext = plugin.file_extension(filename);
    txn.notes.attachment_files.push(filename);
  }

  plugin.compute_and_log_md5sum(connection, ctype, filename, stream);

  if (!filename) return;

  connection.logdebug(plugin, `found attachment file: ${filename}`);
  // See if filename extension matches archive extension list
  if (archives_disabled || !plugin.isArchive(file_ext)) return;

  connection.logdebug(plugin, `found ${file_ext} on archive list`);
  txn.notes.attachment_count++;

  stream.connection = connection;
  stream.pause();

  tmp.file((err, fn, fd) => {
    function cleanup() {
      fs.close(fd, () => {
        connection.logdebug(plugin, `closed fd: ${fd}`);
        fs.unlink(fn, () => {
          connection.logdebug(plugin, `unlinked: ${fn}`);
        });
      });
      stream.resume();
    }
    if (err) {
      txn.notes.attachment_result = [DENYSOFT, err.message];
      connection.logerror(plugin, `Error writing tempfile: ${err.message}`);
      txn.notes.attachment_count--;
      cleanup();
      stream.resume();
      return next();
    }
    connection.logdebug(
      plugin,
      `Got tmpfile: attachment="${filename}" tmpfile="${fn}" fd={fd}`,
    );

    const ws = fs.createWriteStream(fn);
    stream.pipe(ws);
    stream.resume();

    ws.on('error', (error) => {
      txn.notes.attachment_count--;
      txn.notes.attachment_result = [DENYSOFT, error.message];
      connection.logerror(plugin, `stream error: ${error.message}`);
      cleanup();
      next();
    });

    ws.on('close', () => {
      connection.logdebug(plugin, 'end of stream reached');
      connection.pause();
      plugin.unarchive_recursive(connection, fn, filename, (error, files) => {
        txn.notes.attachment_count--;
        cleanup();
        if (err) {
          connection.logerror(plugin, error.message);
          if (err.message === 'maximum archive depth exceeded') {
            txn.notes.attachment_result = [
              DENY,
              'Message contains nested archives exceeding the maximum depth',
            ];
          } else if (/Encrypted file is unsupported/i.test(error.message)) {
            if (!plugin.cfg.main.allow_encrypted_archives) {
              txn.notes.attachment_result = [
                DENY,
                'Message contains encrypted archive',
              ];
            }
          } else if (/Mac metadata is too large/i.test(error.message)) {
            // Skip this error
          } else {
            if (!connection.relaying) {
              txn.notes.attachment_result = [
                DENYSOFT,
                'Error unpacking archive',
              ];
            }
          }
        }

        txn.notes.attachment_archive_files =
          txn.notes.attachment_archive_files.concat(files);
        connection.resume();
        next();
      });
    });
  });
};

exports.hook_data = function (next, connection) {
  const plugin = this;
  if (!connection?.transaction) return next();
  const txn = connection?.transaction;

  txn.parse_body = 1;
  txn.notes.attachment_count = 0;
  txn.notes.attachments = [];
  txn.notes.attachment_ctypes = [];
  txn.notes.attachment_files = [];
  txn.notes.attachment_archive_files = [];
  txn.attachment_hooks((ctype, filename, body, stream) => {
    plugin.start_attachment(connection, ctype, filename, body, stream);
  });
  next();
};

exports.disallowed_extensions = function (txn) {
  const plugin = this;
  if (!plugin.re.bad_extn) return false;

  let bad = false;
  [txn.notes.attachment_files, txn.notes.attachment_archive_files].forEach(
    (items) => {
      if (bad) return;
      if (!items || !Array.isArray(items)) return;
      for (const extn of items) {
        if (!plugin.re.bad_extn.test(extn)) continue;
        bad = extn.split('.').slice(0).pop();
        break;
      }
    },
  );

  return bad;
};

exports.check_attachments = function (next, connection) {
  const txn = connection?.transaction;
  if (!txn) return next();

  // Check for any stored errors from the attachment hooks
  if (txn.notes.attachment_result) {
    const result = txn.notes.attachment_result;
    return next(result[0], result[1]);
  }

  const ctypes = txn.notes.attachment_ctypes;

  // Add in any content type from message body
  const body = txn.body;
  let body_ct;
  if (body && (body_ct = this.re.ct.exec(body.header.get('content-type')))) {
    connection.logdebug(this, `found content type: ${body_ct[1]}`);
    ctypes.push(body_ct[1]);
  }
  // MIME parts
  if (body && body.children) {
    for (let c = 0; c < body.children.length; c++) {
      let child_ct;
      if (
        body.children[c] &&
        (child_ct = this.re.ct.exec(
          body.children[c].header.get('content-type'),
        ))
      ) {
        connection.logdebug(this, `found content type: ${child_ct[1]}`);
        ctypes.push(child_ct[1]);
      }
    }
  }

  const bad_extn = this.disallowed_extensions(txn);
  if (bad_extn) {
    return next(
      DENY,
      `Message contains disallowed file extension (${bad_extn})`,
    );
  }

  const ctypes_result = this.check_items_against_regexps(ctypes, this.re.ctype);
  if (ctypes_result) {
    connection.loginfo(
      this,
      `match ctype="${ctypes_result[0]}" regexp=/${ctypes_result[1]}/`,
    );
    return next(
      DENY,
      `Message contains unacceptable content type (${ctypes_result[0]})`,
    );
  }

  const files = txn.notes.attachment_files;
  const files_result = this.check_items_against_regexps(files, this.re.file);
  if (files_result) {
    connection.loginfo(
      this,
      `match file="${files_result[0]}" regexp=/${files_result[1]}/`,
    );
    return next(
      DENY,
      `Message contains unacceptable attachment (${files_result[0]})`,
    );
  }

  const archive_files = txn.notes.attachment_archive_files;
  const archives_result = this.check_items_against_regexps(
    archive_files,
    this.re.archive,
  );
  if (archives_result) {
    connection.loginfo(
      this,
      `match file="${archives_result[0]}" regexp=/${archives_result[1]}/`,
    );
    return next(
      DENY,
      `Message contains unacceptable attachment (${archives_result[0]})`,
    );
  }

  next();
};

exports.check_items_against_regexps = function (items, regexps) {
  if (!Array.isArray(regexps) || !Array.isArray(items)) return false;
  if (!regexps?.length || !items?.length) return false;

  for (let r = 0; r < regexps.length; r++) {
    for (let i = 0; i < items.length; i++) {
      if (regexps[r].test(items[i])) return [items[i], regexps[r]];
    }
  }
  return false;
};

exports.wait_for_attachment_hooks = (next, connection) => {
  if (connection?.transaction?.notes?.attachment_count > 0) {
    connection.transaction.notes.attachment_next = next;
  } else {
    next();
  }
};