Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for multihead operation #23

Merged
merged 4 commits into from
Dec 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,20 @@ The compose file is set up to mount the data directory to the container where it
Anyway, after running `docker-compose up estimagus`, you should be able to connect to http://localhost:5000, and start exploring the app.


## Single-head vs multihead

Estimagus can operate support more independent views - typically multiple projects, or historical snapshots of a project.
The switch between single- and multi-head operation is the `DATA_DIRS` environmental variable - if supplied, the app will launch in a multi-head mode.
From the usability perspective, accessing the root will always put you on track.


## Configuration

Following environmental variables are recognized and used:

- `SECRET_KEY`: Has to be set to a string, preferably a long and random one. Needed for persistence of user logins.
- `DATA_DIR`: Where the app should look for the data.
- `DATA_DIR`: Where the single-head app should look for the data.
- `DATA_DIRS`: An ordered, comma-separated list of directories with configuration and data of "heads" for the multi-head setup.
- `LOGIN_PROVIDER_NAME`: one of `autologin` or `google`. Autologin logs in any user, google allows Google login when set up properly.
- `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`: When using google login, you need to obtain those from Google to have the Google login working.
- `PLUGINS`: An ordered, comma-separated list of plugin names to load.
Expand Down
15 changes: 15 additions & 0 deletions estimage/inidata.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ class IniAppdata(IniStorage):
PROJECTIVE_QUARTER: str = ""
DAY_INDEX: int = 0
DATADIR: pathlib.Path = pathlib.Path(".")
META: typing.Dict[str, str] = dataclasses.field(default_factory=lambda: dict())

@classmethod
@property
Expand Down Expand Up @@ -147,10 +148,17 @@ def _save_quarters(self, to_save):
retrospective=self.RETROSPECTIVE_QUARTER,
)

def _save_metadata(self, to_save):
to_save["META"] = dict(
description=self.META.get("description", ""),
plugins=self.META.get("plugins_csv", ""),
)

def save(self):
to_save = dict()
self._save_retrospective_period(to_save)
self._save_quarters(to_save)
self._save_metadata(to_save)

with self._manipulate_existing_config(self.CONFIG_FILENAME) as config:
config.update(to_save)
Expand All @@ -163,6 +171,12 @@ def _load_retrospective_period(self, config):
else:
self.RETROSPECTIVE_PERIOD = [datetime.datetime.fromisoformat(s) for s in (start, end)]

def _load_metadata(self, config):
self.META["description"] = config.get(
"META", "description", fallback="")
self.META["plugins_csv"] = config.get(
"META", "plugins", fallback="")

def _load_quarters(self, config):
self.PROJECTIVE_QUARTER = config.get(
"QUARTERS", "projective", fallback=None)
Expand All @@ -179,4 +193,5 @@ def load(cls):
config = result._load_existing_config(cls.CONFIG_FILENAME)
result._load_retrospective_period(config)
result._load_quarters(config)
result._load_metadata(config)
return result
1 change: 1 addition & 0 deletions estimage/plugins/jira/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from ...entities import target
from ... import simpledata
from ...entities import event as evts
from ... import simpledata


JIRA_STATUS_TO_STATE = {
Expand Down
6 changes: 2 additions & 4 deletions estimage/plugins/redhat_compliance/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
import dateutil.relativedelta
import typing

import flask

from ... import simpledata, data, persistence
from ...entities import target
from .. import jira
Expand Down Expand Up @@ -61,7 +59,7 @@ def from_form_and_app(cls, input_form, app) -> "InputSpec":
proj_narrowing = f"sprint = {planning_epoch}"
ret.retrospective_query = " ".join((query_lead, retro_narrowing))
ret.projective_query = " ".join((query_lead, proj_narrowing))
ret.item_class = app.config["classes"]["BaseTarget"]
ret.item_class = app.get_final_class("BaseTarget")
return ret


Expand Down Expand Up @@ -219,7 +217,7 @@ def _get_simple_spec(token):
spec = Spec(
server_url="https://issues.redhat.com",
token=token,
item_class=flask.current_app.config["classes"]["BaseTarget"])
item_class = app.get_final_class("BaseTarget"))
return spec


Expand Down
4 changes: 2 additions & 2 deletions estimage/plugins/redhat_compliance/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
bp = flask.Blueprint("rhcompliance", __name__, template_folder="templates")


@web_utils.is_primary_menu_of(bp, "Red Hat Compliance")
@web_utils.is_primary_menu_of("redhat_compliance", bp, "Red Hat Compliance")
@bp.route('/rhcompliance', methods=("GET", "POST"))
@flask_login.login_required
def sync():
Expand Down Expand Up @@ -41,7 +41,7 @@ def sync():
flask.flash(error_msg)
else:
form.quarter.data = redhat_compliance.datetime_to_epoch(datetime.datetime.today())
next_starts_soon = redhat_compliance.days_to_next_epoch(datetime.datetime.today()) < 20
next_starts_soon = redhat_compliance.days_to_next_epoch(datetime.datetime.today()) < 30
form.project_next.data = next_starts_soon

return web_utils.render_template(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<div class="container">
<div class="row">
<h1>Red Hat Compliance Plugin</h1>
{{ render_form(plugin_form, action=url_for("rhcompliance.sync")) }}
{{ render_form(plugin_form, action=head_url_for("rhcompliance.sync")) }}
</div>
</div>
{% endblock %}
Expand Down
5 changes: 4 additions & 1 deletion estimage/simpledata.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ class IniInDirMixin:
@property
def CONFIG_FILENAME(cls):
try:
datadir = pathlib.Path(flask.current_app.config["DATA_DIR"])
if "head" in flask.current_app.config:
datadir = pathlib.Path(flask.request.blueprints[-1])
else:
datadir = pathlib.Path(flask.current_app.config["DATA_DIR"])
except RuntimeError:
datadir = pathlib.Path(".")
ret = datadir / cls.CONFIG_BASENAME
Expand Down
161 changes: 134 additions & 27 deletions estimage/webapp/__init__.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
import collections
import pathlib
import os

from flask import Flask
import flask
from flask_login import LoginManager
from flask_bootstrap import Bootstrap5
from jinja2 import loaders

from .. import data, simpledata, plugins, PluginResolver
from . import users, config

from .neck import bp as neck_bp
from .main import bp as main_bp
from .vis import bp as vis_bp
from .login import bp as login_bp
Expand All @@ -16,7 +19,7 @@
login = LoginManager()


class PluginFriendlyFlask(Flask):
class PluginFriendlyFlask(flask.Flask):
def __init__(self, import_name, ** kwargs):
webapp_folder = pathlib.Path(__file__).absolute().parent
templates_folder = webapp_folder / "templates"
Expand All @@ -26,46 +29,99 @@ def __init__(self, import_name, ** kwargs):
self.jinja_loader = loaders.FileSystemLoader(
(templates_folder, plugins_folder))

self.template_overrides_map = dict()
self.plugin_resolver = PluginResolver()
self.plugin_resolver.add_known_extendable_classes()
@staticmethod
def _plugin_template_location(plugin_name, template_name):
return str(pathlib.PurePath(plugin_name) / "templates" / template_name)

def get_final_class(self, class_name):
return self.get_config_option("classes").get(class_name)

def get_config_option(self, option):
raise NotImplementedError()

def get_plugins_in_context(self):
raise NotImplementedError()

def get_correct_context_endpoint(self, endpoint):
return endpoint


def set_plugins_dict(self, plugins_dict):
class PluginFriendlySingleheadFlask(PluginFriendlyFlask):
def __init__(self, import_name, ** kwargs):
super().__init__(import_name, ** kwargs)
self._plugin_resolver = PluginResolver()
self._plugin_resolver.add_known_extendable_classes()

def supply_with_plugins(self, plugins_dict):
for plugin in plugins_dict.values():
self.plugin_resolver.resolve_extension(plugin)
self._populate_template_overrides_map(plugins_dict)
self._plugin_resolver.resolve_extension(plugin)

self.config["plugins_templates_overrides"] = self.translate_path
def store_plugins_to_config(self):
self.config["classes"] = self._plugin_resolver.class_dict

def _populate_template_overrides_map(self, plugins_dict):
for plugin_name, plugin in plugins_dict.items():
if not hasattr(plugin, "TEMPLATE_OVERRIDES"):
continue
overrides = plugin.TEMPLATE_OVERRIDES
for overriden, overriding in overrides.items():
template_path = self._plugin_template_location(plugin_name, overriding)
self.template_overrides_map[overriden] = template_path
def get_config_option(self, option):
return self.config[option]

def get_plugins_in_context(self):
return self.get_config_option("PLUGINS")

@staticmethod
def _plugin_template_location(plugin_name, template_name):
return str(pathlib.PurePath(plugin_name) / "templates" / template_name)

def translate_path(self, template_name):
maybe_overriden_path = self.template_overrides_map.get(template_name, template_name)
return maybe_overriden_path
class PluginFriendlyMultiheadFlask(PluginFriendlyFlask):
NON_HEAD_BLUEPRINTS = ("login", "neck")

def __init__(self, import_name, ** kwargs):
super().__init__(import_name, ** kwargs)
self._plugin_resolvers = dict()

def _new_head(self, name):
self._plugin_resolvers[name] = PluginResolver()
self._plugin_resolvers[name].add_known_extendable_classes()

def supply_with_plugins(self, head, plugins_dict):
self._new_head(head)
for plugin in plugins_dict.values():
self._plugin_resolvers[head].resolve_extension(plugin)

def store_plugins_to_config(self, head):
self.config["head"][head]["classes"] = self._plugin_resolvers[head].class_dict

@property
def current_head(self):
return flask.request.blueprints[-1]

def get_config_option(self, option):
return self.config["head"][self.current_head][option]

def get_correct_context_endpoint(self, endpoint):
if (head_name := self.current_head) in self.NON_HEAD_BLUEPRINTS:
return super().get_correct_context_endpoint(endpoint)
return f"{head_name}.{endpoint}"

def get_plugins_in_context(self):
if self.current_head in self.NON_HEAD_BLUEPRINTS:
return dict()
return self.get_config_option("PLUGINS")


def create_app():
if "DATA_DIRS" in os.environ:
app = create_app_multihead()
else:
app = create_app_singlehead()
return app


def create_app(config_class=config.Config):
app = PluginFriendlyFlask(__name__)
def create_app_singlehead(config_class=config.Config):
app = PluginFriendlySingleheadFlask(__name__)
app.jinja_env.globals.update(dict(State=data.State))
app.config.from_object(config_class)
config_class = simpledata.AppData
config_class.DATADIR = pathlib.Path(app.config["DATA_DIR"])
app.config.from_object(config.read_or_create_config(simpledata.AppData))
app.config["classes"] = app.plugin_resolver.class_dict
plugins_dict = {name: plugins.get_plugin(name) for name in app.config["PLUGINS"]}

app.set_plugins_dict(plugins_dict)
app.supply_with_plugins(plugins_dict)
app.store_plugins_to_config()

app.register_blueprint(main_bp)
app.register_blueprint(vis_bp, url_prefix="/vis")
Expand All @@ -85,3 +141,54 @@ def create_app(config_class=config.Config):
pass

return app


def create_app_multihead(config_class=config.MultiheadConfig):
app = PluginFriendlyMultiheadFlask(__name__)
app.jinja_env.globals.update(dict(State=data.State))
app.config.from_object(config_class)
app.config["head"] = collections.defaultdict(dict)

for directory in app.config["DATA_DIRS"]:
configure_head(app, directory)

app.register_blueprint(login_bp)
app.register_blueprint(neck_bp)

Bootstrap5(app)

login.init_app(app)
login.user_loader(users.load_user)
login.login_view = "login.auto_login"

if not app.debug and not app.testing:
pass

return app


def configure_head(app, directory):
config_class = simpledata.AppData
config_class.DATADIR = pathlib.Path(directory)
app.config["head"][directory].update(config.read_or_create_config(simpledata.AppData).__dict__)

metadata = app.config["head"][directory].pop("META", dict())
app.config["head"][directory]["description"] = metadata.get("description", "")
app.config["head"][directory]["PLUGINS"] = config.parse_csv(metadata.get("plugins_csv", ""))

plugins_dict = {name: plugins.get_plugin(name) for name in app.config["head"][directory]["PLUGINS"]}
app.supply_with_plugins(directory, plugins_dict)
app.store_plugins_to_config(directory)

bp = flask.Blueprint(directory, __name__, url_prefix=f"/{directory}")

bp.register_blueprint(main_bp)
bp.register_blueprint(vis_bp, url_prefix="/vis")
bp.register_blueprint(persons_bp)

for plugin in plugins_dict.values():
plugin_bp = plugins.get_plugin_blueprint(plugin)
if plugin_bp:
bp.register_blueprint(plugin_bp, url_prefix=f"/plugins")

app.register_blueprint(bp)
14 changes: 10 additions & 4 deletions estimage/webapp/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,32 @@
from ..simpledata import AppData


def _parse_csv(csv):
def parse_csv(csv):
if csv == "":
return []
else:
return csv.split(",")


class Config:
class CommonConfig:
SECRET_KEY = os.environ.get("SECRET_KEY")
DATA_DIR = os.environ.get("DATA_DIR", "data")
LOGIN_PROVIDER_NAME = os.environ.get("LOGIN_PROVIDER_NAME", "autologin")

GOOGLE_CLIENT_ID = os.environ.get("GOOGLE_CLIENT_ID", None)
GOOGLE_CLIENT_SECRET = os.environ.get("GOOGLE_CLIENT_SECRET", None)
GOOGLE_DISCOVERY_URL = (
"https://accounts.google.com/.well-known/openid-configuration"
)
PLUGINS = _parse_csv(os.environ.get("PLUGINS", ""))


class Config(CommonConfig):
DATA_DIR = os.environ.get("DATA_DIR", "data")
PLUGINS = parse_csv(os.environ.get("PLUGINS", ""))


class MultiheadConfig(CommonConfig):
DATA_DIRS = parse_csv(os.environ.get("DATA_DIRS", "data"))


def read_or_create_config(cls):
config = cls.load()
Expand Down
Loading
Loading