Use ProcessPoolExecutor to speed up link extraction

Author: wpdevelopment11

linkextr.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+from functools import reduce
 from urllib.parse import urlparse
 
 from mistletoe import Document
@@ -8,7 +9,9 @@
 from mistletoe.utils import traverse
 
 import argparse
+import concurrent.futures
 import glob
+import operator
 import os.path
 import sys
 
@@ -74,18 +77,20 @@ def main(args):
     if len(path) == 1 and os.path.isdir(path[0]):
         path = glob.iglob(os.path.join(path[0], "**", "*.md"), recursive=True)
 
-    links = set()
     args = args.__dict__
     if not path:
         lines = sys.stdin.readlines()
         _, lines = frontmatter_split(lines)
         links = findlinks(lines, **args)
     else:
-        for file in path:
-            with open(file, "r", encoding="utf-8") as md:
-                lines = md.readlines()
-            _, lines = frontmatter_split(lines)
-            links |= findlinks(lines, **args)
+        with concurrent.futures.ProcessPoolExecutor(4) as executor:
+            futures = []
+            for file in path:
+                with open(file, "r", encoding="utf-8") as md:
+                    lines = md.readlines()
+                _, lines = frontmatter_split(lines)
+                futures.append(executor.submit(findlinks, lines, **args))
+            links = reduce(operator.or_, [future.result() for future in futures], set())
 
     result = [line + "\n" for line in sorted(links)]
     if output: