diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1675917 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +**/static \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0115abc..9ff1427 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,12 +8,12 @@ WORKDIR qmra ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 -RUN apt update && apt upgrade -y +RUN apt-get update -y && apt upgrade -y && apt-get clean && rm -rf /var/lib/apt/lists/* # install dependencies -RUN pip install --upgrade pip COPY ./requirements.txt . -RUN pip install -r requirements.txt -RUN pip install gunicorn +RUN pip install --upgrade pip &&\ + pip install --no-cache-dir -r requirements.txt &&\ + pip install --no-cache-dir gunicorn # copy project COPY ./qmra ./qmra diff --git a/qmra/management/commands/collect_static_default_entities.py b/qmra/management/commands/collect_static_default_entities.py index 740ea1a..41c5dfe 100644 --- a/qmra/management/commands/collect_static_default_entities.py +++ b/qmra/management/commands/collect_static_default_entities.py @@ -6,6 +6,7 @@ def get_default_pathogens(): pathogen = pd.read_csv("raw_public_data/tbl_pathogen.csv", encoding="windows-1251") + pathogen = pathogen[pathogen.id.isin((3, 32, 34))] health = pd.read_csv("raw_public_data/tbl_health.csv", encoding="windows-1251") # NOTE: HEALTH has been modified 'pathogen_id' is now 'pathogen_group'!! # i.e. Rotavirus -> Viruses, jejuni -> Bacteria, parvum -> Protozoa @@ -27,6 +28,7 @@ def get_default_inflows(): sources = pd.read_csv("raw_public_data/tbl_waterSource.csv", encoding="windows-1251") inflows = pd.merge(inflows, sources, left_on="source_id", right_on="id", how="left").rename(columns={"name": "source_name"}) inflows = pd.merge(inflows, pathogens, left_on="pathogen_id", right_on="id", how="left").rename(columns={"name": "pathogen_name"}) + inflows = inflows[inflows.pathogen_id.isin((3, 32, 34))] return inflows.loc[:, ["source_name", "pathogen_name", "min", "max"]] diff --git a/qmra/risk_assessment/forms.py b/qmra/risk_assessment/forms.py index a9ad40b..3cc2535 100644 --- a/qmra/risk_assessment/forms.py +++ b/qmra/risk_assessment/forms.py @@ -1,14 +1,17 @@ from django import forms from django.core.exceptions import ValidationError -from django.forms import modelformset_factory, HiddenInput -from crispy_forms.bootstrap import AppendedText, Modal +from django.forms import modelformset_factory +from crispy_forms.bootstrap import AppendedText from crispy_forms.helper import FormHelper -from crispy_forms.layout import Layout, Field, Row, Column, Div, HTML, Button +from crispy_forms.layout import Layout, Field, Row, Column, HTML -from qmra.risk_assessment.models import Inflow, DefaultPathogens, DefaultTreatments, Treatment, \ +from qmra.risk_assessment.models import Inflow, DefaultTreatments, Treatment, \ RiskAssessment +def _zero_if_none(x): return x if x is not None else 0 + + class RiskAssessmentForm(forms.ModelForm): class Meta: model = RiskAssessment @@ -23,7 +26,7 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields["source_name"].label = "Select a source type to add inflows" + self.fields["source_name"].label = "Select a source water type to add pathogen concentrations" self.fields['exposure_name'].widget.attrs['min'] = 0 self.fields['volume_per_event'].widget.attrs['min'] = 0 self.helper = FormHelper(self) @@ -53,14 +56,15 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.initial.update(kwargs.get("initial", {})) self.helper = FormHelper(self) self.helper.render_hidden_fields = False self.helper.render_unmentioned_fields = False self.helper.form_tag = False self.helper.disable_csrf = True self.helper.label_class = "text-muted small" - self.fields['pathogen'].choices = DefaultPathogens.choices() - self.fields['pathogen'].label = "Pathogen" + self.fields['pathogen'].disabled = True + self.fields['pathogen'].label = "Reference Pathogen" self.fields['min'].widget.attrs['min'] = 0 self.fields['max'].widget.attrs['min'] = 0 self.fields['min'].label = "Minimum concentration" @@ -75,11 +79,12 @@ def __init__(self, *args, **kwargs): def clean(self): cleaned_data = super().clean() - if cleaned_data["min"] < 0: + mn, mx = cleaned_data.get("min", 0), cleaned_data.get("max", 0) + if mn < 0: self.add_error("min", "this field must be positive or 0") - if cleaned_data["max"] < 0: + if mx < 0: self.add_error("max", "this field must be positive or 0") - if cleaned_data.get("min", 0) > cleaned_data.get("max", 0): + if mn > mx: msg = "minimum concentration must be less than maximum concentration" self.add_error("min", msg) self.add_error("max", msg) @@ -88,16 +93,19 @@ def clean(self): InflowFormSetBase = modelformset_factory( Inflow, form=InflowForm, - extra=0, max_num=30, min_num=0, - can_delete=True, can_delete_extra=True + extra=0, max_num=3, min_num=3, + can_delete=False, can_delete_extra=False ) class InflowFormSet(InflowFormSetBase): - def get_deletion_widget(self): - return forms.CheckboxInput(attrs=dict(label="remove")) def __init__(self, *args, **kwargs): + kwargs["initial"] = [ + {"pathogen": "Rotavirus"}, + {"pathogen": 'Campylobacter jejuni'}, + {"pathogen": "Cryptosporidium parvum"}, + ] super().__init__(*args, **kwargs) self.helper = FormHelper() self.helper.form_tag = False @@ -146,7 +154,7 @@ def __init__(self, *args, **kwargs): self.fields['protozoa_max'].widget.attrs['min'] = 0 label_style = "class='text-muted text-center w-100' style='margin-top: .4em;'" self.helper.layout = Layout( - Field("name", css_class="disabled-input text-center"), + Field("name", css_class="disabled-input d-none"), Row(Column(HTML(f"
")), Column(HTML(f"")), Column(HTML(f""))), @@ -161,24 +169,30 @@ def __init__(self, *args, **kwargs): def clean(self): cleaned_data = super().clean() - if cleaned_data["bacteria_min"] < 0: + b_min = _zero_if_none(cleaned_data.get("bacteria_min", 0)) + b_max = _zero_if_none(cleaned_data.get("bacteria_max", 0)) + v_min = _zero_if_none(cleaned_data.get("viruses_min", 0)) + v_max = _zero_if_none(cleaned_data.get("viruses_max", 0)) + p_min = _zero_if_none(cleaned_data.get("protozoa_min", 0)) + p_max = _zero_if_none(cleaned_data.get("protozoa_max", 0)) + if b_min < 0: self.add_error("bacteria_min", "this field must be positive or 0") - if cleaned_data["bacteria_max"] < 0: + if b_max < 0: self.add_error("bacteria_max", "this field must be positive or 0") - if cleaned_data["viruses_min"] < 0: + if v_min < 0: self.add_error("viruses_min", "this field must be positive or 0") - if cleaned_data["viruses_max"] < 0: + if v_max < 0: self.add_error("viruses_max", "this field must be positive or 0") - if cleaned_data["protozoa_min"] < 0: + if p_min < 0: self.add_error("protozoa_min", "this field must be positive or 0") - if cleaned_data["protozoa_max"] < 0: + if p_max < 0: self.add_error("protozoa_max", "this field must be positive or 0") msg = "min. must be less than max" - if cleaned_data.get("bacteria_min", 0) > cleaned_data.get("bacteria_max", 0): + if b_min > b_max: self.add_error("bacteria_min", msg) - if cleaned_data.get("viruses_min", 0) > cleaned_data.get("viruses_max", 0): + if v_min > v_max: self.add_error("viruses_min", msg) - if cleaned_data.get("protozoa_min", 0) > cleaned_data.get("protozoa_max", 0): + if p_min > p_max: self.add_error("protozoa_min", msg) return cleaned_data diff --git a/qmra/risk_assessment/migrations/0002_alter_inflow_pathogen_and_more.py b/qmra/risk_assessment/migrations/0002_alter_inflow_pathogen_and_more.py new file mode 100644 index 0000000..0087d1f --- /dev/null +++ b/qmra/risk_assessment/migrations/0002_alter_inflow_pathogen_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.0.6 on 2024-10-18 14:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('risk_assessment', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='inflow', + name='pathogen', + field=models.CharField(choices=[('', '---------'), ('Bacteria', [('Campylobacter jejuni', 'Campylobacter jejuni')]), ('Viruses', [('Rotavirus', 'Rotavirus')]), ('Protozoa', [('Cryptosporidium parvum', 'Cryptosporidium parvum')])], max_length=256), + ), + migrations.AlterField( + model_name='riskassessmentresult', + name='dalys_risk', + field=models.CharField(choices=[('min', 'min'), ('max', 'max'), ('none', 'none')], max_length=4), + ), + migrations.AlterField( + model_name='riskassessmentresult', + name='infection_risk', + field=models.CharField(choices=[('min', 'min'), ('max', 'max'), ('none', 'none')], max_length=4), + ), + migrations.AlterField( + model_name='riskassessmentresult', + name='pathogen', + field=models.CharField(choices=[('', '---------'), ('Bacteria', [('Campylobacter jejuni', 'Campylobacter jejuni')]), ('Viruses', [('Rotavirus', 'Rotavirus')]), ('Protozoa', [('Cryptosporidium parvum', 'Cryptosporidium parvum')])], max_length=256), + ), + ] diff --git a/qmra/risk_assessment/models.py b/qmra/risk_assessment/models.py index 05fab04..2a87692 100644 --- a/qmra/risk_assessment/models.py +++ b/qmra/risk_assessment/models.py @@ -198,7 +198,8 @@ class DefaultSources(StaticEntity): @classmethod def choices(cls): - grouped = {grp: list(v) for grp, v in groupby(sorted(cls.data.values(), key=lambda x: x.name), key=lambda x: x.name.split(",")[0])} + grouped = {grp: list(v) for grp, v in + groupby(sorted(cls.data.values(), key=lambda x: x.name), key=lambda x: x.name.split(",")[0])} return [ ("", "---------"), *[(k, [(x.name, x.name) for x in v]) for k, v in grouped.items()], @@ -294,7 +295,8 @@ class DefaultExposures(StaticEntity): @classmethod def choices(cls): - grouped = {grp: list(v) for grp, v in groupby(sorted(cls.data.values(), key=lambda x: x.name), key=lambda x: x.name.split(",")[0])} + grouped = {grp: list(v) for grp, v in + groupby(sorted(cls.data.values(), key=lambda x: x.name), key=lambda x: x.name.split(",")[0])} return [ ("", "---------"), *[(k, [(x.name, x.name) for x in sorted(v, key=lambda x: x.name)]) for k, v in grouped.items()], @@ -322,7 +324,8 @@ class RiskAssessment(models.Model): @property def infection_risk(self): - return any(r.infection_risk for r in self.results.all()) + risks = {r.infection_risk for r in self.results.all()} + return 'max' if 'max' in risks else ("min" if 'min' in risks else 'none') @property def dalys_risk(self): @@ -336,6 +339,9 @@ def pathogens_labels(self): def treatments_labels(self): return ", ".join([treatment.name for treatment in self.treatments.all()]) + def results_list(self): + return [r.as_dict() for r in self.results.all()] + def __str__(self): return self.name @@ -344,10 +350,8 @@ class RiskAssessmentResult(models.Model): risk_assessment = models.ForeignKey(RiskAssessment, on_delete=models.CASCADE, related_name="results") pathogen = models.CharField(choices=DefaultPathogens.choices(), max_length=256) - - infection_risk = models.BooleanField() - dalys_risk = models.BooleanField() - + infection_risk = models.CharField(choices=[("min", "min"), ("max", "max"), ("none", "none")], max_length=4) + dalys_risk = models.CharField(choices=[("min", "min"), ("max", "max"), ("none", "none")], max_length=4) infection_minimum_lrv_min = models.FloatField() infection_minimum_lrv_max = models.FloatField() infection_minimum_lrv_q1 = models.FloatField() @@ -358,7 +362,6 @@ class RiskAssessmentResult(models.Model): infection_maximum_lrv_q1 = models.FloatField() infection_maximum_lrv_q3 = models.FloatField() infection_maximum_lrv_median = models.FloatField() - dalys_minimum_lrv_min = models.FloatField() dalys_minimum_lrv_max = models.FloatField() dalys_minimum_lrv_q1 = models.FloatField() @@ -369,3 +372,31 @@ class RiskAssessmentResult(models.Model): dalys_maximum_lrv_q1 = models.FloatField() dalys_maximum_lrv_q3 = models.FloatField() dalys_maximum_lrv_median = models.FloatField() + + def as_dict(self): + return dict( + ra_name=self.risk_assessment.name, + pathogen=self.pathogen, + infection_risk=self.infection_risk, + dalys_risk=self.dalys_risk, + infection_minimum_lrv_min=self.infection_minimum_lrv_min, + infection_minimum_lrv_max=self.infection_minimum_lrv_max, + infection_minimum_lrv_q1=self.infection_minimum_lrv_q1, + infection_minimum_lrv_q3=self.infection_minimum_lrv_q3, + infection_minimum_lrv_median=self.infection_minimum_lrv_median, + infection_maximum_lrv_min=self.infection_maximum_lrv_min, + infection_maximum_lrv_max=self.infection_maximum_lrv_max, + infection_maximum_lrv_q1=self.infection_maximum_lrv_q1, + infection_maximum_lrv_q3=self.infection_maximum_lrv_q3, + infection_maximum_lrv_median=self.infection_maximum_lrv_median, + dalys_minimum_lrv_min=self.dalys_minimum_lrv_min, + dalys_minimum_lrv_max=self.dalys_minimum_lrv_max, + dalys_minimum_lrv_q1=self.dalys_minimum_lrv_q1, + dalys_minimum_lrv_q3=self.dalys_minimum_lrv_q3, + dalys_minimum_lrv_median=self.dalys_minimum_lrv_median, + dalys_maximum_lrv_min=self.dalys_maximum_lrv_min, + dalys_maximum_lrv_max=self.dalys_maximum_lrv_max, + dalys_maximum_lrv_q1=self.dalys_maximum_lrv_q1, + dalys_maximum_lrv_q3=self.dalys_maximum_lrv_q3, + dalys_maximum_lrv_median=self.dalys_maximum_lrv_median + ) diff --git a/qmra/risk_assessment/plots.py b/qmra/risk_assessment/plots.py index 4498213..a163918 100644 --- a/qmra/risk_assessment/plots.py +++ b/qmra/risk_assessment/plots.py @@ -3,11 +3,33 @@ from plotly import express as px import pandas as pd +RISK_CATEGORY_BG_COLORS = dict( + none='#E2FBAC', min='#FFDDB5', max='#FFECF4' +) +MAX_COLOR_SEQ = ["#FF0532", + "#FF506F", + "#FF8DA2" + ] -def risk_plots(risk_assessment_results): +MIN_COLOR_SEQ = [ + "#FF873F", + "#FFA570", + "#ED5500" +] + +NONE_COLOR_SEQ = [ + "#1B6638", + "#46A16A", + "#88D0A5" +] + +COLOR_SEQS = dict(min=MIN_COLOR_SEQ, max=MAX_COLOR_SEQ, none=NONE_COLOR_SEQ) + + +def risk_plots(risk_assessment_results, risk_category="none"): infection_prob_fig = go.Figure() dalys_fig = go.Figure() - for r in risk_assessment_results: + for i, r in enumerate(risk_assessment_results): infection_prob_fig.add_trace(go.Box( x=["Minimum LRV", "Maximum LRV"], lowerfence=[r.infection_minimum_lrv_min, r.infection_maximum_lrv_min], @@ -16,6 +38,7 @@ def risk_plots(risk_assessment_results): q3=[r.infection_minimum_lrv_q3, r.infection_maximum_lrv_q3], median=[r.infection_minimum_lrv_median, r.infection_maximum_lrv_median], name=r.pathogen, + marker=dict(color=COLOR_SEQS[r.infection_risk][i % 3]) )) dalys_fig.add_trace(go.Box( x=["Minimum LRV", "Maximum LRV"], @@ -25,15 +48,17 @@ def risk_plots(risk_assessment_results): q3=[r.dalys_minimum_lrv_q3, r.dalys_maximum_lrv_q3], median=[r.dalys_minimum_lrv_median, r.dalys_maximum_lrv_median], name=r.pathogen, + marker=dict(color=COLOR_SEQS[r.dalys_risk][i % 3]) )) infection_prob_fig.update_layout( boxmode='group', # font_family="Helvetica Neue, Helvetica, Arial, sans-serif", font_color="black", - # plot_bgcolor=bgcolor, + plot_bgcolor=RISK_CATEGORY_BG_COLORS[risk_category], xaxis=dict(title="", showgrid=False), - yaxis=dict(title="Probability of infection per year", showgrid=False), + yaxis=dict(title="Probability of infection per year", + showgrid=False), margin=dict(l=0, r=0, t=0, b=0), legend=dict( orientation="h", @@ -48,8 +73,9 @@ def risk_plots(risk_assessment_results): text="tolerable level", textposition="end", yanchor="top", - ) - # line=dict(color=lcolor, width=3) + font=dict(color="rgb(0, 3, 226)") + ), + line=dict(color="rgb(0, 3, 226)", width=3) ) infection_prob_fig.update_traces( marker_size=8 @@ -59,7 +85,7 @@ def risk_plots(risk_assessment_results): boxmode='group', # font_family="Helvetica Neue, Helvetica, Arial, sans-serif", font_color="black", - # plot_bgcolor=bgcolor, + plot_bgcolor=RISK_CATEGORY_BG_COLORS[risk_category], xaxis=dict(title="", showgrid=False), yaxis=dict(title="DALYs pppy", showgrid=False), margin=dict(l=0, r=0, t=0, b=0), @@ -76,8 +102,9 @@ def risk_plots(risk_assessment_results): text="tolerable level", textposition="end", yanchor="top", - ) - # line=dict(color=lcolor, width=3) + font=dict(color="rgb(0, 3, 226)") + ), + line=dict(color="rgb(0, 3, 226)", width=3) ) dalys_fig.update_traces( marker_size=8 @@ -86,86 +113,3 @@ def risk_plots(risk_assessment_results): return plot(infection_prob_fig, output_type="div", config={'displayModeBar': False}, include_plotlyjs=False), \ plot(dalys_fig, output_type="div", config={'displayModeBar': False}, include_plotlyjs=False) - -def inflows_plot(inflows): - df = pd.DataFrame.from_records([i.__dict__ for i in inflows]) - # print(df.head()) - # reshaping dataframe for plotting - df_inflow2 = pd.melt( - df, - ("pathogen",), value_vars=("min", "max") - ) - - df_inflow2 = df_inflow2.rename( - columns={"pathogen": "Pathogen", "variable": ""} - ) - fig2 = px.bar( - df_inflow2, - x="", - y="value", - log_y=True, - facet_col="Pathogen", - barmode="group", - # color_discrete_sequence=risk_colors_extended, - ) - - fig2.for_each_annotation( - lambda a: a.update( - text=a.text.split("=")[-1], font=dict(size=10, color="black"), - ) - ) - - fig2.update_layout( - autosize=True, - # font_family="Helvetica Neue, Helvetica, Arial, sans-serif", - font_color="black", - yaxis_title="Source water concentrations in N/L", - margin=dict(l=0, r=0, t=40, b=0), - ) - - return plot(fig2, output_type="div", config={'displayModeBar': False, "responsive": True}, include_plotlyjs=False) - - -def treatments_plot(treatments): - # reshaping - df = pd.DataFrame.from_records([i.__dict__ for i in treatments]) - # print(df.head().loc[:, ["bacteria_min", "name", "viruses_max"]]) - df = pd.melt(df, - ("name",), value_vars=( - "bacteria_min", "bacteria_max", "viruses_min", "viruses_max", "protozoa_min", "protozoa_max" - ), - var_name="metric" - ) - splitted = df.metric.str.split("_", expand=True) - df["Pathogen Group"] = splitted[0] - df[""] = splitted[1] - df = df.rename( - columns={ - "name": "Treatment", - } - ) - # print(df.head()) - fig = px.bar( - df, - x="", - y="value", - color="Treatment", - facet_col="Pathogen Group", - category_orders={"Pathogen Group": ["viruses", "bacteria", "protozoa"]}, - # color_discrete_sequence=risk_colors_extended, - ) - - fig.for_each_annotation( - lambda a: a.update(text=a.text.split("=")[-1], font=dict(size=13)) - ) - fig.update_layout( - margin=dict(l=0, r=0, t=20, b=0), - legend=dict(orientation="h", yanchor="top", - xanchor="center", - x=0.5, ), - # font_family="Helvetica Neue, Helvetica, Arial, sans-serif", - font_color="black", - yaxis_title="Logremoval of individual treatment step", - ) - - return plot(fig, output_type="div", config={'displayModeBar': False}, include_plotlyjs=False) diff --git a/qmra/risk_assessment/risk.py b/qmra/risk_assessment/risk.py index ac6f6c2..f8534e7 100644 --- a/qmra/risk_assessment/risk.py +++ b/qmra/risk_assessment/risk.py @@ -15,8 +15,8 @@ def get_annual_risk( n_years: int = 1000, ): inflow = np.random.normal( - loc=(np.log10(inflow_min + 10 ** (-8)) + np.log10(inflow_max))/2, - scale=(np.log10(inflow_max) - np.log10(inflow_min + 10 ** (-8)))/4, + loc=(np.log10(inflow_min + 10 ** (-8)) + np.log10(inflow_max)) / 2, + scale=(np.log10(inflow_max) - np.log10(inflow_min + 10 ** (-8))) / 4, size=n_events ) lrv = np.ones((n_events,)) * log_removal @@ -89,8 +89,9 @@ def to_dalys(pr, pat=pathogen): # make result results[inflow.pathogen] = RiskAssessmentResult( risk_assessment=risk_assessment, - infection_risk=(min_prob_mean + max_prob_mean) > (10 ** -4), - dalys_risk=to_dalys(min_prob_mean+max_prob_mean) > (10 ** -6), + infection_risk="max" if min_prob_mean > (10 ** -4) else ("min" if max_prob_mean > (10 ** -4) else "none"), + dalys_risk="max" if to_dalys(min_prob_mean) > (10 ** -6) else ( + "min" if to_dalys(max_prob_mean) > (10 ** -6) else "none"), pathogen=inflow.pathogen, infection_maximum_lrv_min=maximum_lrv_min, infection_maximum_lrv_max=maximum_lrv_max, diff --git a/qmra/risk_assessment/templates/assessment-configurator.html b/qmra/risk_assessment/templates/assessment-configurator.html index 7506a20..629795c 100644 --- a/qmra/risk_assessment/templates/assessment-configurator.html +++ b/qmra/risk_assessment/templates/assessment-configurator.html @@ -2,35 +2,72 @@ {% load crispy_forms_tags %} {% load static %} {% block body %} -