formatting & minor fixes

Author: huettenhain

refinery/lib/patterns/__init__.py

@@ -163,7 +163,7 @@ def __getattr__(self, name):
     R'-----END(?:\s[A-Z0-9]+)+-----'
 ).format(n=R'(?:\r\n|\n\r|\n)', b=R'[0-9a-zA-Z\+\/]')
 
-__all__ = [ 
+__all__ = [
     'pattern',
     'alphabet',
     'tokenize',

refinery/lib/patterns/tlds.py

@@ -1,4 +1,3 @@
-
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 tlds = ['xn\\-\\-vermgensberatung\\-pwb',

refinery/units/crypto/keyderive/__init__.py

@@ -24,6 +24,7 @@ class HashAlgorithms(Enum):
     MD2 = MD2
     MD4 = MD4
     MD5 = MD5
+    SHA1 = SHA1
     SHA256 = SHA256
     SHA512 = SHA512
     SHA224 = SHA224

refinery/units/formats/__init__.py

@@ -27,14 +27,12 @@ class PathExtractorUnit(Unit):
 
     @classmethod
     def interface(cls, argp):
-        argp.add_argument('paths', metavar='path', nargs='*', default=['*'], type=pathspec,
-            help=(
-                'A path from which data is to be extracted. Each item is returned '
-                ' as a separate output of this unit. Paths may contain wildcards. '
-                'The default is a single asterix, which means that every item will '
-                'be extracted.'
-            )
-        )
+        argp.add_argument('paths', metavar='path', nargs='*', default=['*'], type=pathspec, help=(
+            'A path from which data is to be extracted. Each item is returned '
+            ' as a separate output of this unit. Paths may contain wildcards. '
+            'The default is a single asterix, which means that every item will '
+            'be extracted.'
+        ))
         return super().interface(argp)
 
     def _check_reachable(self, path: str) -> bool:

refinery/units/formats/deserialize_java.py

@@ -9,6 +9,7 @@
 
 from .. import Unit
 
+
 class JavaEncoder(json.JSONEncoder):
 
     def encode(self, obj):
@@ -24,6 +25,7 @@ def default(self, obj):
                 return str(obj)
             raise
 
+
 class dsjava(Unit):
     """
     Deserialize Java serialized data and re-serialize as JSON.

refinery/units/formats/pe/pemeta.py

@@ -4,7 +4,7 @@
 
 from functools import lru_cache
 from contextlib import suppress
-from pefile import PE
+from pefile import PE, DIRECTORY_ENTRY
 from datetime import datetime, timezone
 from asn1crypto import cms
 from asn1crypto import x509
@@ -22,8 +22,8 @@ class pemeta(Unit):
     - Timestamps
     - If present, .NET header information
     """
-    def __init__(self,
-        all : arg('-c', '--custom',
+    def __init__(
+        self, all : arg('-c', '--custom',
             help='Unless enabled, everything will be extracted.') = True,
         version    : arg('-V', help='Parse the VERSION resource.') = False,
         timestamps : arg('-T', help='Extract time stamps.') = False,
@@ -50,7 +50,7 @@ def _parse_pedict(self, bin):
 
     @lru_cache(maxsize=1, typed=False)
     def _getpe(self, data: bytearray) -> PE:
-        return PE(data=data)
+        return PE(data=data, fast_load=True)
 
     def parse_signature(self, data: bytearray) -> dict:
         """
@@ -130,7 +130,9 @@ def parse_file_info(self, data: bytearray) -> dict:
         the version resource of an input PE file, if available.
         """
         try:
-            FileInfoList = self._getpe(data).FileInfo
+            pe = self._getpe(data)
+            pe.parse_data_directories(directories=[DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE']])
+            FileInfoList = pe.FileInfo
         except AttributeError:
             return None
         for FileInfo in FileInfoList:
@@ -163,6 +165,13 @@ def dt(ts):
             return datetime.fromtimestamp(ts, tz=timezone.utc).replace(tzinfo=None)
 
         pe = self._getpe(data)
+        pe.parse_data_directories(directories=[
+            DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_IMPORT'],
+            DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXPORT'],
+            DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_DEBUG'],
+            DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE']
+        ])
+
         info = {}
 
         with suppress(AttributeError):

refinery/units/meta/couple.py

@@ -28,8 +28,9 @@ def __init__(
             type=str,
             help='Part of an arbitrary command line to be executed.',
             metavar='token'),
-        buffer  : arg.switch('-b', help='Buffer the command output for one execution rather than streaming it.') = False,
-        timeout : arg('-t', metavar='T', help='Set an execution timeout as a floating point number in seconds, there is none by default.') = 0.0
+        buffer: arg.switch('-b', help='Buffer the command output for one execution rather than streaming it.') = False,
+        timeout: arg('-t', metavar='T',
+            help='Set an execution timeout as a floating point number in seconds, there is none by default.') = 0.0
     ) -> Unit: pass
 
     def process(self, data):

refinery/units/meta/emit.py

@@ -38,6 +38,7 @@ def process(self, data):
 
     @classmethod
     def run(cls, argv=None, stream=None):
-        super(emit, cls).run(argv=argv,
+        super(emit, cls).run(
+            argv=argv,
             stream=stream or open(__import__('os').devnull, 'rb')
         )

refinery/units/obfuscation/ps1/securestring.py

@@ -3,7 +3,6 @@
 import re
 
 from .. import Deobfuscator
-from . import string_quote, string_escape
 from ...crypto.cipher.secstr import secstr
 from ...blockwise.pack import pack
 from ....lib.patterns import formats
@@ -43,7 +42,6 @@ def _decrypt_block(self, data, match):
             self._secstr.args.key = self._pack(match.group(5).encode(self.codec))
         decoded = self._secstr(match.group(2).encode(self.codec))
         decoded = decoded.decode(self.codec)
-        #result = string_quote(string_escape(decoded))
         result = F'\n\n{decoded}\n\n'
         brackets = match.group(6).count(')')
         start = match.start()

refinery/units/pattern/__init__.py

@@ -72,7 +72,7 @@ def __call__(self, match):
     try:
         fmt = fmt.decode('UNICODE_ESCAPE')
     except AttributeError:
-        if type(fmt) != str:
+        if not isinstance(fmt, str):
             raise
 
     fmt = fmt.encode('UTF8')
@@ -272,9 +272,10 @@ def __init__(
         self, regex : arg(type=regexp, help='Regular expression to match.'),
         # TODO: Use positional only in Python 3.8
         # /,
-        multiline   : arg.switch('-M', help='caret and dollar match the beginning and end of a line, the dot does not match line breaks.') = False,
-        ignorecase  : arg.switch('-I', help='ignore capitalization for alphabetic characters.') = False,
-        utf16       : arg.switch('-u', help='search for unicode patterns instead of ascii.') = False,
+        multiline   : arg.switch('-M', help='Caret and dollar match the beginning and end of '
+                                            'a line, a dot does not match line breaks.') = False,
+        ignorecase  : arg.switch('-I', help='Ignore capitalization for alphabetic characters.') = False,
+        utf16       : arg.switch('-u', help='Search for unicode patterns instead of ascii.') = False,
         min=1, max=None, len=None, whitespace=False, unique=False, longest=False, take=None
     ):
         super().__init__(