Merge branch 'release/2.8.0'

Makefile

@@ -80,3 +80,12 @@ dist: clean ## builds source and wheel package
 
 install: clean ## install the package to the active Python's site-packages
  python setup.py install
+
+debug-iter-topology:
+ mkdir /tmp/logs/ 2>/dev/null || echo /tmp/logs/ already exist
+ sparse run \
+ -n spamscope_debug_iter \
+ -e debug \
+ -o topology.max.spout.pending=1 \
+ -o "topology.sleep.spout.wait.strategy.time.ms=10" \
+ -o "topology.tick.tuple.freq.secs=10"

README.md

@@ -177,10 +177,13 @@ You can use:
  * [Ansible](./ansible/README.md): to install and run SpamScope on server
 
 # Topologies
-SpamScope comes with three topologies:
+SpamScope comes with six topologies:
  - [spamscope_debug](./topologies/spamscope_debug.py): the output are JSON files on file system.
  - [spamscope_elasticsearch](./topologies/spamscope_elasticsearch.py): the output are stored in Elasticsearch indexes.
  - [spamscope_redis](./topologies/spamscope_redis.py): the output are stored in Redis.
+ - [spamscope_debug_iter](./topologies/spamscope_debug_iter.py): It uses generator to send mails in topology. The output are JSON files on file system.
+ - [spamscope_elasticsearch_iter](./topologies/spamscope_elasticsearch_iter.py): It uses generator to send mails in topology. The output are stored in Elasticsearch indexes.
+ - [spamscope_redis_iter](./topologies/spamscope_redis_iter.py): It uses generator to send mails in topology. The output are stored in Redis.
 
 If you want submit SpamScope topology use `spamscope-topology submit` tool. For more details [see SpamScope cli tools](src/cli/README.md):
 

README.rst

@@ -291,13 +291,21 @@ run SpamScope on server
 Topologies
 ==========
 
-SpamScope comes with three topologies: -
+SpamScope comes with six topologies: -
 `spamscope_debug <./topologies/spamscope_debug.py>`__: the output are
 JSON files on file system. -
 `spamscope_elasticsearch <./topologies/spamscope_elasticsearch.py>`__:
 the output are stored in Elasticsearch indexes. -
 `spamscope_redis <./topologies/spamscope_redis.py>`__: the output are
-stored in Redis.
+stored in Redis. -
+`spamscope_debug_iter <./topologies/spamscope_debug_iter.py>`__: It uses
+generator to send mails in topology. The output are JSON files on file
+system. -
+`spamscope_elasticsearch_iter <./topologies/spamscope_elasticsearch_iter.py>`__:
+It uses generator to send mails in topology. The output are stored in
+Elasticsearch indexes. -
+`spamscope_redis_iter <./topologies/spamscope_redis_iter.py>`__: It uses
+generator to send mails in topology. The output are stored in Redis.
 
 If you want submit SpamScope topology use ``spamscope-topology submit``
 tool. For more details `see SpamScope cli tools <src/cli/README.md>`__:

conf/spamscope.example.yml

@@ -49,6 +49,50 @@ files-mails:
  path_mails: /path/mails2
 
 
+# Spout file on file system
+# Use an iterator. Safe for RAM
+iter-files-mails:
+
+ # The mails in processing older that fail.after.seconds will be failed
+ fail.after.seconds: 60
+
+ # Post processing
+ post_processing:
+
+ # move or remove mails analyzed, default remove
+ what: remove
+
+ # Where you want move the analyzed mails, default /tmp/moved
+ where: /tmp/moved
+
+ # Where you want move the failed mails, default /tmp/failed
+ where.failed: /tmp/failed
+
+ # Mailboxes
+ mailboxes:
+ test:
+ mail_server: hostname
+ # Trust string is used to get sender IP address from mail server.
+ # More details:
+ # https://github.com/SpamScope/mail-parser/blob/v0.4.6/mailparser/__init__.py#L221
+ trust_string: "test_trust_string"
+ files_pattern: "*untroubled*"
+ path_mails: /path/mails1
+
+ # This flag enables Outlook msg parsing for every mails in mailbox
+ # Default value is false
+ outlook: false
+
+ # List of others headers to get
+ headers:
+ - custom_one
+ - custom_two
+ test1:
+ mail_server: hostname
+ trust_string: "test1_trust_string"
+ files_pattern: "*"
+ path_mails: /path/mails2
+
 # Bolts configurations
 # Phishing bolt configuration
 phishing:

project.clj

@@ -1,4 +1,4 @@
-(defproject spamscope "2.7.0-SNAPSHOT"
+(defproject spamscope "2.8.0-SNAPSHOT"
  :resource-paths ["_resources"]
  :target-path "_build"
  :min-lein-version "2.0.0"

src/bolts/tokenizer.py

@@ -24,7 +24,7 @@
 import random
 import six
 from collections import deque
-from cPickle import BadPickleGet
+from cPickle import PickleError
 
 from streamparse import Stream
 import mailparser
@@ -85,7 +85,7 @@ def load_filters(self):
  try:
  obj = load_obj(path)
  setattr(self, "analyzed_" + i, obj)
- except (IOError, EOFError, ValueError, BadPickleGet):
+ except (IOError, EOFError, ValueError, PickleError):
  setattr(self, "analyzed_" + i, deque(
  maxlen=getattr(self, "maxlen_" + i)))
 

src/cli/spamscope_topology.py

@@ -57,8 +57,14 @@ def get_args():
  submit.add_argument(
  "-g",
  "--topology",
- choices=["spamscope_debug", "spamscope_elasticsearch",
- "spamscope_redis"],
+ choices=[
+ "spamscope_debug_iter",
+ "spamscope_debug",
+ "spamscope_elasticsearch_iter",
+ "spamscope_elasticsearch",
+ "spamscope_redis_iter",
+ "spamscope_redis",
+ ],
  default="debug",
  help="SpamScope topology.",
  dest="topology")

src/modules/attachments/attachments.py

@@ -353,9 +353,11 @@ def withhashes(cls, attachments=[]):
  try:
  payload = base64.b64decode(i["payload"])
  except TypeError, e:
- payload = base64.b64decode(i["payload"] + "===")
- i.setdefault("errors", []).append(repr(e))
-
+ try:
+ payload = base64.b64decode(i["payload"] + "===")
+ i.setdefault("errors", []).append(repr(e))
+ except TypeError:
+ continue
  else:
  payload = i["payload"]
 

src/modules/attachments/thug_analysis.py

@@ -58,8 +58,12 @@ def generate_json_report():
  if m is None:
  return
 
- report = json.loads(m(tempfile.gettempdir()))
- return report
+ try:
+ report = json.loads(m(tempfile.gettempdir()))
+ except TypeError:
+ return
+ else:
+ return report
 
 
 class CustomWatchdog(Watchdog):

src/options.py

@@ -19,7 +19,7 @@
 
 from os.path import join
 
-__version__ = "2.7.0"
+__version__ = "2.8.0"
 __configuration_path__ = "/etc/spamscope"
 
 __defaults__ = {

src/spouts/README.md

@@ -0,0 +1,13 @@
+# Overview
+In this folder there are all SpamScope `spouts`.
+
+# How add a new spout
+These are the steps to add a new `spout` to Spamscope:
+
+ - add a new module in [spouts](./) folder. This module should implement a new class that has `AbstractSpout` as base.
+
+ - import the new class in [\_\_init\_\_.py](./__init__.py)
+
+ - add the new section in [main configuration file](../../conf/spamscope.example.yml). The name of this section will be used in topology file
+
+ - add the new spout in [topology](../../topologies) 

src/spouts/__init__.py

@@ -15,3 +15,4 @@
 """
 
 from .files_mails import FilesMailSpout
+from .iter_files_mails import IterFilesMailSpout

src/spouts/files_mails.py

@@ -128,17 +128,21 @@ def next_tuple(self):
  self.log("EMITTED - {!r}".format(mail_string))
 
  processing = mail.filename + ".processing"
- shutil.move(mail.filename, processing)
-
- self.emit([
- processing, # 0
- mail.mail_server, # 1
- mail.mailbox, # 2
- mail.priority, # 3
- mail.trust, # 4
- mail.mail_type, # 5
- mail.headers], # 6
- tup_id=mail.filename)
+
+ try:
+ shutil.move(mail.filename, processing)
+ except IOError:
+ self.log("ALREADY EMITTED - {!r}".format(mail_string))
+ else:
+ self.emit([
+ processing, # 0
+ mail.mail_server, # 1
+ mail.mailbox, # 2
+ mail.priority, # 3
+ mail.trust, # 4
+ mail.mail_type, # 5
+ mail.headers], # 6
+ tup_id=mail.filename)
 
  def ack(self, tup_id):
  """Acknowledge tup_id, that is the path_mail. """