allow guilds to have multable instances + some edits

Pipfile

@@ -8,12 +8,9 @@ flask = "*"
 gevent = "*"
 natural = "*"
 dateutils = "*"
-pymongo = {extras = ["srv"], version = "*"}
 click = "*"
 python-dotenv = "*"
 requests-oauthlib = "*"
-flask-discord = {editable = true, ref = "v9", git = "https://github.com/weibeu/Flask-Discord.git"}
-sentry-sdk = {extras = ["flask"], version = "*"}
 
 [dev-packages]
 
@@ -23,3 +20,18 @@ python_version = "3.9"
 [scripts]
 web = "python main.py serve --no-debug"
 webd = "python main.py serve --debug"
+secret = "python main.py secretkey"
+gensecret = "python main.py secretkey --new"
+
+[packages.pymongo]
+extras = [ "srv",]
+version = "*"
+
+[packages.flask-discord]
+editable = true
+ref = "v9"
+git = "https://github.com/weibeu/Flask-Discord.git"
+
+[packages.sentry-sdk]
+extras = [ "flask",]
+version = "*"

example.env

@@ -0,0 +1,15 @@
+PORT=5214
+HOST=127.0.0.1
+
+# sentry dsn url
+SENTRY_DSN =
+# randomkey (you can generate this with pipenv run secret --new)
+FLASK_SECRET_KEY =
+
+DISCORD_CLIENT_ID =
+DISCORD_CLIENT_SECRET =
+DISCORD_CLIENT_REDIRECT_URI = http://localhost:5214/api/auth/discord/callback
+
+# if guild has 2 or more instances you can label them with _1 _2 _3 etc (dont set 0 as thats what no label guilds)
+MONGO_URI_gid=mongodburl
+MONGO_URI_gid_1=mongodburl1

logviewer2/constants.py

@@ -2,9 +2,13 @@
 
 
 class Constants:
-    RE_ENV_DISCORD = re.compile("DISCORD_(.*)")
-    RE_ENV_MONGODB = re.compile("MONGO_URI_(.*)")
-    RE_FPROXY_FACE = re.compile("/fproxy/(.*)")
+    RE_ENV_DISCORD = re.compile(r"DISCORD_(.*)")
+    RE_ENV_MONGODB = re.compile(r"MONGO_URI_(.*)")
+    RE_FPROXY_FACE = re.compile(r"/fproxy/(.*)")
     DISCORD_ASSISTS_URL = 'https://discordapp.com/assets/'
     LOG_FORMAT = '[%(levelname)s] %(asctime)s - %(name)s:%(lineno)d - %(message)s'
-    FONTS = {"FONT_WHITNEY_LIGHT": "6c6374bad0b0b6d204d8d6dc4a18d820.woff", "FONT_WHITNEY_NORMAL":"e8acd7d9bf6207f99350ca9f9e23b168.woff", "FONT_WHITNEY_MEDIUM":"3bdef1251a424500c1b3a78dea9b7e57.woff", "FONT_WHITNEYMEDIUM_MEDIUM":"be0060dafb7a0e31d2a1ca17c0708636.woff", "FONT_WHITNEY_BOLD":"8e12fb4f14d9c4592eb8ec9f22337b04.woff"}
+    FONTS = {"FONT_WHITNEY_LIGHT": "6c6374bad0b0b6d204d8d6dc4a18d820.woff",
+             "FONT_WHITNEY_NORMAL": "e8acd7d9bf6207f99350ca9f9e23b168.woff",
+             "FONT_WHITNEY_MEDIUM": "3bdef1251a424500c1b3a78dea9b7e57.woff",
+             "FONT_WHITNEYMEDIUM_MEDIUM": "be0060dafb7a0e31d2a1ca17c0708636.woff",
+             "FONT_WHITNEY_BOLD": "8e12fb4f14d9c4592eb8ec9f22337b04.woff"}

logviewer2/utils/__init__.py

@@ -1,3 +1,6 @@
+from logviewer2.constants import Constants
+
+
 def GET_SECRET_KEY(config):
     key = config.get("FLASK_SECRET_KEY", None)
     if not key:
@@ -15,17 +18,34 @@ def GET_REV():
     return str(REV, 'utf=8')[:7]
 
 
+def GET_INSTANCE(qid):
+    if not type(qid) is str:
+        raise ValueError("incorrect itemtype")
+
+    if qid.isdigit():
+        return int(qid), 0
+    elif qid:
+        if "_" in qid:
+            try:
+                parts = list(map(int, qid.split('_')))
+                return parts[0], parts[1]
+            except ValueError:
+                raise ValueError("one or both types are invalid (not ints)")
+        else:
+            raise ValueError("Incorrect string format")
+
+
 def DOWNLOAD_FONTS():
     import tempfile
     import requests
 
     from pathlib import Path
-    from logviewer2.constants import Constants
 
     FDIR = tempfile.TemporaryDirectory()
     pFDIR = Path(FDIR.name)
     for name, file in Constants.FONTS.items():
         with requests.get(Constants.DISCORD_ASSISTS_URL + file, stream=True) as r:
+            # noinspection PyBroadException
             try:
                 r.raise_for_status()
                 r.encoding = r.apparent_encoding

logviewer2/utils/db.py

@@ -1,5 +1,7 @@
 from dotenv import dotenv_values
 from pymongo import MongoClient
+
+from logviewer2.utils import GET_INSTANCE
 from logviewer2.utils.regexcfg import GET_MCONFIG
 
 
@@ -10,7 +12,12 @@ def __init__(self):
         self.dbs_conns = dict()
 
         for (gid, connURI) in self.dbs.items():
-            self.dbs_conns[gid] = MongoClient(connURI).modmail_bot
+            gid, instance = GET_INSTANCE(gid)
+            if gid not in self.dbs_conns:
+                self.dbs_conns[gid] = dict()
+
+            self.dbs_conns[gid].update({instance: MongoClient(connURI).modmail_bot})
 
-    def get(self, gid):
-        return self.dbs_conns.get(int(gid), None)
+    def get(self, gid, instance_id):
+        ginstances = self.dbs_conns.get(gid, dict())
+        return ginstances.get(instance_id, None)

logviewer2/utils/decos.py

@@ -4,14 +4,20 @@
 from oauthlib.oauth2 import InvalidClientError, TokenExpiredError
 
 from logviewer2.log_utils.models import LogEntry
+from logviewer2.utils import GET_INSTANCE
 
 
 def with_logs_evidence(fn):
     @wraps(fn)
     def decorated_view(*args, **kwargs):
-        gid = kwargs['gid']
+        gid, instid = None, None
+        try:
+            gid, instid = GET_INSTANCE(kwargs['qid'])
+        except ValueError:
+            abort(404)
+
         logkey = kwargs['logkey']
-        db = current_app.db.get(gid)
+        db = current_app.db.get(gid, instid)
         if not db:
             abort(404)
 
@@ -34,9 +40,15 @@ def decorated_view(*args, **kwargs):
 def with_logs(fn):
     @wraps(fn)
     def decorated_view(*args, **kwargs):
-        gid = kwargs['gid']
+        try:
+            gid, instid = GET_INSTANCE(kwargs['qid'])
+        except ValueError:
+            gid, instid = None, None
+            abort(404)
+
         logkey = kwargs['logkey']
-        db = current_app.db.get(gid)
+
+        db = current_app.db.get(gid, instid)
         if not db:
             abort(404)
 

logviewer2/utils/regexcfg.py

@@ -16,5 +16,5 @@ def GET_MCONFIG(cfg):
     for (key, value) in cfg.items():
         match = re.match(Constants.RE_ENV_MONGODB, key)
         if match:
-            obj[int(match.groups()[0])] = value
+            obj[match.groups()[0]] = value
     return obj

logviewer2/web.py

@@ -7,7 +7,7 @@
 
 from werkzeug.middleware.proxy_fix import ProxyFix
 
-from logviewer2.utils import GET_SECRET_KEY, DOWNLOAD_FONTS
+from logviewer2.utils import DOWNLOAD_FONTS, GET_SECRET_KEY
 from logviewer2.utils.decos import with_logs_evidence, with_logs, with_user
 from logviewer2.utils.db import DB
 from logviewer2.utils.regexcfg import GET_DCONFIG
@@ -87,22 +87,22 @@ def robotstxt():
     return r
 
 
-@app.route("/<int:gid>/<logkey>")
+@app.route("/<string:qid>/<logkey>")
 @with_user
 @with_logs
-def logviewer_render(gid, logkey):
+def logviewer_render(qid, logkey):
     return g.document.render_html(user=g.user)
 
 
-@app.route("/evidence/<int:gid>/<logkey>")
+@app.route("/evidence/<string:qid>/<logkey>")
 @with_user
 @with_logs_evidence
-def logviewer_render_evidence(gid, logkey):
+def logviewer_render_evidence(qid, logkey):
     return g.document.render_html(user=g.user)
 
 
 # API killme
-@app.route("/api/raw/<int:gid>/<logkey>")
+@app.route("/api/raw/<string:qid>/<logkey>")
 @with_logs
-def api_raw_render(gid, logkey):
+def api_raw_render(qid, logkey):
     return Response(g.document.render_plain_text(), mimetype="text/plain"), 200

main.py

@@ -1,4 +1,4 @@
-from gevent import monkey;
+from gevent import monkey
 
 monkey.patch_all()
 
@@ -19,7 +19,6 @@
 import click
 from dotenv import dotenv_values
 from werkzeug.serving import run_simple
-from logviewer2.web import app
 
 
 @click.group()
@@ -42,6 +41,7 @@ def serve(debug):
         integrations=[FlaskIntegration(), ExcepthookIntegration(always_run=True)],
         traces_sample_rate=1.0
     )
+    from logviewer2.web import app
 
     if debug:
         app.debug = True
@@ -51,5 +51,15 @@ def serve(debug):
         return run_simple(config.get("HOST", "localhost"), int(config.get("PORT", "5214")), app, threaded=True)
 
 
+@cli.command()
+@click.option('--new/--no-new', '-n', default=False)
+def secretkey(new):
+    from logviewer2.utils import GET_SECRET_KEY
+    key = GET_SECRET_KEY({} if new else dotenv_values(".env"))
+    if not new:
+        print(key.decode("utf-8"))
+    return key if not new else ""
+
+
 if __name__ == '__main__':
     cli()

package.json

@@ -8,7 +8,9 @@
   "license": "MIT",
   "dependencies": {
     "animate.css": "^4.1.1",
+    "core-js": "^3.19.0",
     "highlight": "^0.2.4",
-    "jquery": "3.3.1"
+    "jquery": "3.3.1",
+    "materialize-css": "^1.0.0"
   }
 }

templates/base.html

@@ -27,9 +27,9 @@
 
 		<!--  Scripts-->
 
-		<script src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
+		<script src="/static/modules/jquery/dist/jquery.min.js"></script>
 		<script src="https://cdnjs.cloudflare.com/ajax/libs/core-js/2.4.1/core.js"></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/js/materialize.min.js"></script>
+        <script src="/static/modules/materialize-css/dist/js/materialize.min.js"></script>
         <script>
             M.AutoInit();
             $('.dropdown-trigger').dropdown({coverTrigger: false});

yarn.lock

@@ -7,6 +7,11 @@ animate.css@^4.1.1:
   resolved "https://registry.yarnpkg.com/animate.css/-/animate.css-4.1.1.tgz#614ec5a81131d7e4dc362a58143f7406abd68075"
   integrity sha512-+mRmCTv6SbCmtYJCN4faJMNFVNN5EuCTTprDTAo7YzIGji2KADmakjVA3+8mVDkZ2Bf09vayB35lSQIex2+QaQ==
 
+core-js@^3.19.0:
+  version "3.19.0"
+  resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.19.0.tgz#9e40098a9bc326c7e81b486abbd5e12b9d275176"
+  integrity sha512-L1TpFRWXZ76vH1yLM+z6KssLZrP8Z6GxxW4auoCj+XiViOzNPJCAuTIkn03BGdFe6Z5clX5t64wRIRypsZQrUg==
+
 highlight@^0.2.4:
   version "0.2.4"
   resolved "https://registry.yarnpkg.com/highlight/-/highlight-0.2.4.tgz#8ac02875b03f5935e0675852b76cfe1fd58e0dff"
@@ -16,3 +21,8 @@ [email protected]:
   version "3.3.1"
   resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.3.1.tgz#958ce29e81c9790f31be7792df5d4d95fc57fbca"
   integrity sha512-Ubldcmxp5np52/ENotGxlLe6aGMvmF4R8S6tZjsP6Knsaxd/xp3Zrh50cG93lR6nPXyUFwzN3ZSOQI0wRJNdGg==
+
+materialize-css@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/materialize-css/-/materialize-css-1.0.0.tgz#8d5db1c4a81c6d65f3b2e2ca83a8e08daa24d1be"
+  integrity sha512-4/oecXl8y/1i8RDZvyvwAICyqwNoKU4or5uf8uoAd74k76KzZ0Llym4zhJ5lLNUskcqjO0AuMcvNyDkpz8Z6zw==