From c4c8e74fd8a042dff46f4bd8051355c2acee73eb Mon Sep 17 00:00:00 2001 From: nboyse Date: Thu, 25 Jan 2024 12:20:40 +0000 Subject: [PATCH 01/24] initial implementation of EU data report and upload feature --- reports/forms.py | 5 + reports/jinja2/generics/table.jinja | 21 +++- reports/jinja2/reports/index.jinja | 7 +- .../jinja2/reports/upload_report_csv.jinja | 23 ++++ reports/migrations/0003_eudatamodel.py | 29 +++++ reports/migrations/0004_auto_20240125_1134.py | 104 ++++++++++++++++ reports/models.py | 20 ++- reports/reports/eu_data.py | 61 +++++++++ reports/urls.py | 5 + reports/views.py | 117 +++++++++++++++++- 10 files changed, 384 insertions(+), 8 deletions(-) create mode 100644 reports/forms.py create mode 100644 reports/jinja2/reports/upload_report_csv.jinja create mode 100644 reports/migrations/0003_eudatamodel.py create mode 100644 reports/migrations/0004_auto_20240125_1134.py create mode 100644 reports/reports/eu_data.py diff --git a/reports/forms.py b/reports/forms.py new file mode 100644 index 000000000..523df0907 --- /dev/null +++ b/reports/forms.py @@ -0,0 +1,5 @@ +from django import forms + + +class UploadCSVForm(forms.Form): + file = forms.FileField() diff --git a/reports/jinja2/generics/table.jinja b/reports/jinja2/generics/table.jinja index 1d5bcc2a2..84f35e9da 100644 --- a/reports/jinja2/generics/table.jinja +++ b/reports/jinja2/generics/table.jinja @@ -64,6 +64,25 @@ {% else -%} {{ govukTable({ "head": report.headers(), - "rows": report.rows() + "rows": report.rows(current_page_data), + "pagination": current_page_data.paginator }) }} + + {% endif -%} \ No newline at end of file diff --git a/reports/jinja2/reports/index.jinja b/reports/jinja2/reports/index.jinja index 091523221..70bac7ec4 100644 --- a/reports/jinja2/reports/index.jinja +++ b/reports/jinja2/reports/index.jinja @@ -13,12 +13,11 @@

Reports Index

+ Upload CSV report You will find a list of reports below that can be viewed.
{{ govukTable({ "head": report.headers(), "rows": report.rows() }) }}
-{% endblock %} - - - + +{% endblock %} \ No newline at end of file diff --git a/reports/jinja2/reports/upload_report_csv.jinja b/reports/jinja2/reports/upload_report_csv.jinja new file mode 100644 index 000000000..817a3bdc1 --- /dev/null +++ b/reports/jinja2/reports/upload_report_csv.jinja @@ -0,0 +1,23 @@ +{% extends "layouts/layout.jinja" %} +{% from "components/table/macro.njk" import govukTable %} + +{% set page_title = 'Upload CSV' %} + +{% block breadcrumb %} + {{ breadcrumbs(request, [ + {'text': "Upload CSV"} + ]) }} +{% endblock %} + +{% block content %} +

+ Upload CSV +

+ Select the type of file to upload. + +
+ + {{ form.as_p() }} + +
+{% endblock %} \ No newline at end of file diff --git a/reports/migrations/0003_eudatamodel.py b/reports/migrations/0003_eudatamodel.py new file mode 100644 index 000000000..015d8b4e7 --- /dev/null +++ b/reports/migrations/0003_eudatamodel.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.23 on 2024-01-23 16:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("reports", "0002_create_custom_permissions"), + ] + + operations = [ + migrations.CreateModel( + name="EUDataModel", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("workbasket_id", models.IntegerField()), + ("name", models.CharField(max_length=255)), + ("date", models.DateField()), + ], + ), + ] diff --git a/reports/migrations/0004_auto_20240125_1134.py b/reports/migrations/0004_auto_20240125_1134.py new file mode 100644 index 000000000..292c967ce --- /dev/null +++ b/reports/migrations/0004_auto_20240125_1134.py @@ -0,0 +1,104 @@ +# Generated by Django 3.2.23 on 2024-01-25 11:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("reports", "0003_eudatamodel"), + ] + + operations = [ + migrations.RemoveField( + model_name="eudatamodel", + name="date", + ), + migrations.RemoveField( + model_name="eudatamodel", + name="name", + ), + migrations.RemoveField( + model_name="eudatamodel", + name="workbasket_id", + ), + migrations.AddField( + model_name="eudatamodel", + name="add_code", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="duty", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="end_date", + field=models.DateField(blank=True, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="geographical_area_exists", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="goods_code", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="goods_nomenclature_exists", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="legal_base", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="meas_type_code", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="measure_exists", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="measure_type", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="measure_type_exists", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="order_no", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="origin", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="origin_code", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="red_ind", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="eudatamodel", + name="start_date", + field=models.DateField(blank=True, null=True), + ), + ] diff --git a/reports/models.py b/reports/models.py index aace44019..9ada0b99d 100644 --- a/reports/models.py +++ b/reports/models.py @@ -3,5 +3,23 @@ class Report(models.Model): class Meta: - # Define the name for the database table (optional) db_table = "report" + + +class EUDataModel(models.Model): + goods_code = models.CharField(max_length=255, null=True, blank=True) + add_code = models.CharField(max_length=255, null=True, blank=True) + order_no = models.CharField(max_length=255, null=True, blank=True) + start_date = models.DateField(null=True, blank=True) + end_date = models.DateField(null=True, blank=True) + red_ind = models.CharField(max_length=255, null=True, blank=True) + origin = models.CharField(max_length=255, null=True, blank=True) + measure_type = models.CharField(max_length=255, null=True, blank=True) + legal_base = models.CharField(max_length=255, null=True, blank=True) + duty = models.CharField(max_length=255, null=True, blank=True) + origin_code = models.CharField(max_length=255, null=True, blank=True) + meas_type_code = models.CharField(max_length=255, null=True, blank=True) + goods_nomenclature_exists = models.CharField(max_length=255, null=True, blank=True) + geographical_area_exists = models.CharField(max_length=255, null=True, blank=True) + measure_type_exists = models.CharField(max_length=255, null=True, blank=True) + measure_exists = models.CharField(max_length=255, null=True, blank=True) diff --git a/reports/reports/eu_data.py b/reports/reports/eu_data.py new file mode 100644 index 000000000..1f0215976 --- /dev/null +++ b/reports/reports/eu_data.py @@ -0,0 +1,61 @@ +from reports.reports.base_table import ReportBaseTable +from reports.models import EUDataModel + +from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage + + +class Report(ReportBaseTable): + name = "Table report of EU data" + enabled = True + + headers_list = [ + "goods_code", + "add_code", + "order_no", + "start_date", + "end_date", + "red_ind", + "origin", + "measure_type", + "legal_base", + "duty", + "origin_code", + "meas_type_code", + "goods_nomenclature_exists", + "geographical_area_exists", + "measure_type_exists", + "measure_exists", + ] + + def headers(self) -> [dict]: + return [ + {"text": header.replace("_", " ").capitalize()} + for header in self.headers_list + ] + + def row(self, row) -> [dict]: + return [{"text": str(getattr(row, field, None))} for field in self.headers_list] + + def rows(self, current_page_data) -> [[dict]]: + table_rows = [] + for row in current_page_data: + table_rows.append(self.row(row)) + + return table_rows + + def query(self): + return EUDataModel.objects.all() + + def get_paginated_data(self, page=1, items_per_page=25): + report_data = self.query() + + paginator = Paginator(report_data, items_per_page) + + try: + current_page_data = paginator.page(page) + except PageNotAnInteger: + current_page_data = paginator.page(1) + except EmptyPage: + current_page_data = paginator.page(paginator.num_pages) + + return current_page_data diff --git a/reports/urls.py b/reports/urls.py index f625a0bc6..78dcd15ce 100644 --- a/reports/urls.py +++ b/reports/urls.py @@ -22,6 +22,11 @@ views.export_report_to_csv, name="export_report_with_tabs_to_csv", ), + path( + "reports/upload_csv", + views.upload_report_csv, + name="upload_report_csv", + ), ] for report in utils.get_reports(): diff --git a/reports/views.py b/reports/views.py index 608a4cd76..42f3a7876 100644 --- a/reports/views.py +++ b/reports/views.py @@ -1,8 +1,11 @@ import csv +import pandas as pd +from datetime import datetime from django.contrib.auth.decorators import permission_required from django.http import HttpResponse -from django.shortcuts import render +from django.shortcuts import render, redirect +from io import StringIO from openpyxl import Workbook from openpyxl.chart import BarChart, Reference @@ -10,6 +13,9 @@ import reports.utils as utils +from reports.forms import UploadCSVForm +from reports.models import EUDataModel + @permission_required("reports.view_report_index") def index(request): @@ -22,11 +28,18 @@ def index(request): @permission_required("reports.view_report") def report(request): - # find the report based on the request report_class = utils.get_report_by_slug(request.resolver_match.url_name) + # Adjust the number of items per page as needed + items_per_page = 25 + + current_page_data = report_class().get_paginated_data( + request.GET.get("page", 1), items_per_page + ) + context = { "report": report_class(), + "current_page_data": current_page_data, } return render( @@ -145,3 +158,103 @@ def export_report_to_excel(request, report_slug): workbook.save(response) return response + + +def upload_report_csv(request): + if request.method == "POST": + form = UploadCSVForm(request.POST, request.FILES) + if form.is_valid(): + uploaded_file = form.cleaned_data["file"] + + if is_eu_data_file(uploaded_file): + content = read_excel_as_csv(uploaded_file) + save_eu_data(content) + return redirect("reports:table_report_of_eu_data") + + else: + form = UploadCSVForm() + + return render(request, "reports/upload_report_csv.jinja", {"form": form}) + + +def is_eu_data_file(uploaded_file): + try: + content = read_excel_as_csv(uploaded_file) + csv_reader = csv.DictReader(StringIO(content)) + file_columns = set(csv_reader.fieldnames) if csv_reader.fieldnames else set() + + print(f"File Columns: {file_columns}") + return True + except Exception as e: + print(f"Error reading CSV content: {e}") + return False + + +def read_excel_as_csv(uploaded_file): + try: + df = pd.read_excel(uploaded_file) + csv_data = df.to_csv(index=False, encoding="utf-8") + return csv_data + except Exception as e: + print(f"Error converting Excel to CSV: {e}") + return "" + + +def save_eu_data(content): + try: + csv_reader = csv.DictReader(StringIO(content)) + for row in csv_reader: + print(f"Row: {row}") + try: + start_date_str = row.get("Start date", None) + end_date_str = row.get("End date", None) + + # Convert date strings to the correct format "YYYY-MM-DD" + start_date = ( + datetime.strptime(start_date_str, "%Y-%m-%d %H:%M:%S").strftime( + "%Y-%m-%d" + ) + if start_date_str + else None + ) + end_date = ( + datetime.strptime(end_date_str, "%Y-%m-%d %H:%M:%S").strftime( + "%Y-%m-%d" + ) + if end_date_str + else None + ) + + instance = EUDataModel( + goods_code=row.get("Goods code", None), + add_code=row.get("Add code", None), + order_no=row.get("Order No.", None), + start_date=start_date, + end_date=end_date, + red_ind=row.get("RED_IND", None), + origin=row.get("Origin", None), + measure_type=row.get("Measure type", None), + legal_base=row.get("Legal base", None), + duty=row.get("Duty", None), + origin_code=row.get("Origin code", None), + meas_type_code=row.get("Meas. type code", None), + goods_nomenclature_exists=row.get( + "Goods Nomenclature Exists in TAP", None + ), + geographical_area_exists=row.get( + "Geographical Area Exists in TAP", None + ), + measure_type_exists=row.get("Measure Type Exists in TAP", None), + measure_exists=row.get("Measure Exists in TAP", None), + ) + + print(f"Instance to be saved: {instance}") + instance.save() + + print(f"Instance saved: {instance}") + except Exception as e: + print(f"Error saving row {row}: {e}") + + except Exception as e: + print(f"Error reading CSV content: {e}") + pass From b7d92a4e1601ef7448ea28e93d1e63f00eb69539 Mon Sep 17 00:00:00 2001 From: nboyse Date: Thu, 25 Jan 2024 15:06:47 +0000 Subject: [PATCH 02/24] stylistic changes and fixing export/other reports --- reports/jinja2/generics/table.jinja | 101 +++++++++++++++--- reports/jinja2/reports/index.jinja | 4 +- reports/jinja2/reports/report_table.jinja | 4 +- .../jinja2/reports/upload_report_csv.jinja | 8 +- reports/reports/base.py | 3 + reports/reports/base_table.py | 2 +- .../blank_goods_nomenclature_descriptions.py | 2 +- reports/reports/quotas_cannot_be_used.py | 20 +++- reports/urls.py | 2 +- reports/views.py | 20 ++-- 10 files changed, 132 insertions(+), 34 deletions(-) diff --git a/reports/jinja2/generics/table.jinja b/reports/jinja2/generics/table.jinja index 84f35e9da..d71839e41 100644 --- a/reports/jinja2/generics/table.jinja +++ b/reports/jinja2/generics/table.jinja @@ -27,7 +27,7 @@

{{ report.tab_name }}

- Export to CSV + Export to CSV {{ govukTable({ "head": report.headers(), "rows": report.rows() @@ -65,24 +65,93 @@ {{ govukTable({ "head": report.headers(), "rows": report.rows(current_page_data), - "pagination": current_page_data.paginator }) }} - + + + {% endif -%} {% endif -%} \ No newline at end of file diff --git a/reports/jinja2/reports/index.jinja b/reports/jinja2/reports/index.jinja index 70bac7ec4..482655a67 100644 --- a/reports/jinja2/reports/index.jinja +++ b/reports/jinja2/reports/index.jinja @@ -13,7 +13,9 @@

Reports Index

- Upload CSV report + {% if request.user.is_superuser %} + Upload EU CSV file + {% endif %} You will find a list of reports below that can be viewed.
diff --git a/reports/jinja2/reports/report_table.jinja b/reports/jinja2/reports/report_table.jinja index 24e5ea9ce..d9fb29ec3 100644 --- a/reports/jinja2/reports/report_table.jinja +++ b/reports/jinja2/reports/report_table.jinja @@ -13,12 +13,12 @@ Report: {{ report.name }} {% if not report.tab_name2 %} - Export to CSV + Export to CSV {% endif %}

{{ report.description|safe }}

-
+
{% include "generics/table.jinja" %}
{% endblock %} diff --git a/reports/jinja2/reports/upload_report_csv.jinja b/reports/jinja2/reports/upload_report_csv.jinja index 817a3bdc1..2ff410d30 100644 --- a/reports/jinja2/reports/upload_report_csv.jinja +++ b/reports/jinja2/reports/upload_report_csv.jinja @@ -11,13 +11,13 @@ {% block content %}

- Upload CSV + Upload EU CSV file

- Select the type of file to upload. + Select the file to upload that will replace the current EU data report.
- {{ form.as_p() }} - +
{{ form.as_p() }}
+
{% endblock %} \ No newline at end of file diff --git a/reports/reports/base.py b/reports/reports/base.py index 0633b4a1e..c02ee9bf7 100644 --- a/reports/reports/base.py +++ b/reports/reports/base.py @@ -18,3 +18,6 @@ def slug(cls): result = slugify(cls.name).replace("-", "_") result = result.replace("__", "_") return result + + def get_paginated_data(self, page, items_per_page): + pass \ No newline at end of file diff --git a/reports/reports/base_table.py b/reports/reports/base_table.py index b638c9353..e32d52b34 100644 --- a/reports/reports/base_table.py +++ b/reports/reports/base_table.py @@ -29,7 +29,7 @@ def headers(self) -> [dict]: pass @abstractmethod - def rows(self) -> [dict]: + def rows(self, current_page_data=None) -> [dict]: pass @abstractmethod diff --git a/reports/reports/blank_goods_nomenclature_descriptions.py b/reports/reports/blank_goods_nomenclature_descriptions.py index 37e2d6c80..a45dd5eb7 100644 --- a/reports/reports/blank_goods_nomenclature_descriptions.py +++ b/reports/reports/blank_goods_nomenclature_descriptions.py @@ -39,7 +39,7 @@ def row(self, row: GoodsNomenclatureDescription) -> [dict]: {"text": live_description}, ] - def rows(self) -> [[dict]]: + def rows(self, current_page_data) -> [[dict]]: table_rows = [] for row in self.query(): table_rows.append(self.row(row)) diff --git a/reports/reports/quotas_cannot_be_used.py b/reports/reports/quotas_cannot_be_used.py index b83d3694f..bcaa1f1bb 100644 --- a/reports/reports/quotas_cannot_be_used.py +++ b/reports/reports/quotas_cannot_be_used.py @@ -1,5 +1,6 @@ import datetime +from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage from django.db.models import Exists, Q from reports.reports.base_table import ReportBaseTable @@ -34,9 +35,9 @@ def row(self, row: QuotaDefinition) -> [dict]: {"text": row.reason}, ] - def rows(self) -> [[dict]]: + def rows(self, current_page_data) -> [[dict]]: table_rows = [] - for row in self.query(): + for row in current_page_data: table_rows.append(self.row(row)) return table_rows @@ -134,3 +135,18 @@ def find_quotas_that_cannot_be_used(self, quotas_with_definition_periods): quota_order_number.reason = "Definition period has not been set" return list(matching_data) + + + def get_paginated_data(self, page=1, items_per_page=25): + report_data = self.query() + + paginator = Paginator(report_data, items_per_page) + + try: + current_page_data = paginator.page(page) + except PageNotAnInteger: + current_page_data = paginator.page(1) + except EmptyPage: + current_page_data = paginator.page(paginator.num_pages) + + return current_page_data \ No newline at end of file diff --git a/reports/urls.py b/reports/urls.py index 78dcd15ce..bbc263c40 100644 --- a/reports/urls.py +++ b/reports/urls.py @@ -8,7 +8,7 @@ urlpatterns = [ path("reports/", views.index, name="index"), path( - "reports//export-to-csv", + "reports///export-to-csv", views.export_report_to_csv, name="export_report_to_csv", ), diff --git a/reports/views.py b/reports/views.py index 42f3a7876..c4ee5cd89 100644 --- a/reports/views.py +++ b/reports/views.py @@ -3,6 +3,7 @@ from datetime import datetime from django.contrib.auth.decorators import permission_required +from django.core.paginator import Paginator from django.http import HttpResponse from django.shortcuts import render, redirect from io import StringIO @@ -49,7 +50,7 @@ def report(request): ) -def export_report_to_csv(request, report_slug, current_tab=None): +def export_report_to_csv(request, report_slug, current_tab=None, current_page=None): report_class = utils.get_report_by_slug(report_slug) report_instance = report_class() @@ -65,7 +66,7 @@ def export_report_to_csv(request, report_slug, current_tab=None): tab_mapping = { report_instance.tab_name: ( report_instance.headers(), - report_instance.rows(), + report_instance.rows(current_page_data=current_page), ), report_instance.tab_name2: ( report_instance.headers2(), @@ -96,7 +97,11 @@ def export_report_to_csv(request, report_slug, current_tab=None): headers = ( report_instance.headers() if hasattr(report_instance, "headers") else None ) - rows = report_instance.rows() if hasattr(report_instance, "rows") else None + if current_page: + current_page_data = Paginator(report_instance.query(), 25).page(current_page) + rows = report_instance.rows(current_page_data=current_page_data) if hasattr(report_instance, "rows") else None + else: + rows = report_instance.rows() if hasattr(report_instance, "rows") else None writer = csv.writer(response) @@ -117,7 +122,6 @@ def export_report_to_csv(request, report_slug, current_tab=None): return response - def export_report_to_excel(request, report_slug): report_class = utils.get_report_by_slug(report_slug) report_instance = report_class() @@ -168,6 +172,10 @@ def upload_report_csv(request): if is_eu_data_file(uploaded_file): content = read_excel_as_csv(uploaded_file) + + # Clear existing data in EUDataTable + EUDataModel.objects.all().delete() + save_eu_data(content) return redirect("reports:table_report_of_eu_data") @@ -233,11 +241,11 @@ def save_eu_data(content): end_date=end_date, red_ind=row.get("RED_IND", None), origin=row.get("Origin", None), - measure_type=row.get("Measure type", None), + measure_type=row.get(" Measure type", None), legal_base=row.get("Legal base", None), duty=row.get("Duty", None), origin_code=row.get("Origin code", None), - meas_type_code=row.get("Meas. type code", None), + meas_type_code=row.get(" Meas. type code", None), goods_nomenclature_exists=row.get( "Goods Nomenclature Exists in TAP", None ), From 5ff48fa446931dab6ecda6a4322872a92e96bab0 Mon Sep 17 00:00:00 2001 From: nboyse Date: Thu, 25 Jan 2024 15:42:15 +0000 Subject: [PATCH 03/24] adjust margins, add new url pattern for current page --- reports/jinja2/generics/table.jinja | 2 +- reports/jinja2/reports/report_table.jinja | 8 ++++++-- reports/reports/base.py | 2 +- reports/reports/eu_data.py | 3 ++- reports/reports/quotas_cannot_be_used.py | 3 +-- reports/urls.py | 7 ++++++- reports/views.py | 17 ++++++++++++++--- 7 files changed, 31 insertions(+), 11 deletions(-) diff --git a/reports/jinja2/generics/table.jinja b/reports/jinja2/generics/table.jinja index d71839e41..87291247b 100644 --- a/reports/jinja2/generics/table.jinja +++ b/reports/jinja2/generics/table.jinja @@ -27,7 +27,7 @@

{{ report.tab_name }}

- Export to CSV + Export to CSV {{ govukTable({ "head": report.headers(), "rows": report.rows() diff --git a/reports/jinja2/reports/report_table.jinja b/reports/jinja2/reports/report_table.jinja index d9fb29ec3..727baea6b 100644 --- a/reports/jinja2/reports/report_table.jinja +++ b/reports/jinja2/reports/report_table.jinja @@ -13,12 +13,16 @@ Report: {{ report.name }} {% if not report.tab_name2 %} - Export to CSV + {% if current_page_data %} + Export to CSV + {% else %} + Export to CSV + {% endif %} {% endif %}

{{ report.description|safe }}

-
+
{% include "generics/table.jinja" %}
{% endblock %} diff --git a/reports/reports/base.py b/reports/reports/base.py index c02ee9bf7..cc90d66b7 100644 --- a/reports/reports/base.py +++ b/reports/reports/base.py @@ -20,4 +20,4 @@ def slug(cls): return result def get_paginated_data(self, page, items_per_page): - pass \ No newline at end of file + pass diff --git a/reports/reports/eu_data.py b/reports/reports/eu_data.py index 1f0215976..9bc10df13 100644 --- a/reports/reports/eu_data.py +++ b/reports/reports/eu_data.py @@ -6,6 +6,7 @@ class Report(ReportBaseTable): name = "Table report of EU data" + description = "Imported data from the EU tariff using the upload CSV feature" enabled = True headers_list = [ @@ -44,7 +45,7 @@ def rows(self, current_page_data) -> [[dict]]: return table_rows def query(self): - return EUDataModel.objects.all() + return EUDataModel.objects.all().order_by("goods_code") def get_paginated_data(self, page=1, items_per_page=25): report_data = self.query() diff --git a/reports/reports/quotas_cannot_be_used.py b/reports/reports/quotas_cannot_be_used.py index bcaa1f1bb..f859e5314 100644 --- a/reports/reports/quotas_cannot_be_used.py +++ b/reports/reports/quotas_cannot_be_used.py @@ -136,7 +136,6 @@ def find_quotas_that_cannot_be_used(self, quotas_with_definition_periods): return list(matching_data) - def get_paginated_data(self, page=1, items_per_page=25): report_data = self.query() @@ -149,4 +148,4 @@ def get_paginated_data(self, page=1, items_per_page=25): except EmptyPage: current_page_data = paginator.page(paginator.num_pages) - return current_page_data \ No newline at end of file + return current_page_data diff --git a/reports/urls.py b/reports/urls.py index bbc263c40..bbfa2b987 100644 --- a/reports/urls.py +++ b/reports/urls.py @@ -8,10 +8,15 @@ urlpatterns = [ path("reports/", views.index, name="index"), path( - "reports///export-to-csv", + "reports//export-to-csv", views.export_report_to_csv, name="export_report_to_csv", ), + path( + "reports///export-to-csv", + views.export_report_to_csv, + name="export_report_with_page_to_csv", + ), path( "reports//export-to-excel", views.export_report_to_excel, diff --git a/reports/views.py b/reports/views.py index c4ee5cd89..b0efff339 100644 --- a/reports/views.py +++ b/reports/views.py @@ -98,10 +98,20 @@ def export_report_to_csv(request, report_slug, current_tab=None, current_page=No report_instance.headers() if hasattr(report_instance, "headers") else None ) if current_page: - current_page_data = Paginator(report_instance.query(), 25).page(current_page) - rows = report_instance.rows(current_page_data=current_page_data) if hasattr(report_instance, "rows") else None + current_page_data = Paginator(report_instance.query(), 25).page( + current_page + ) + rows = ( + report_instance.rows(current_page_data=current_page_data) + if hasattr(report_instance, "rows") + else None + ) else: - rows = report_instance.rows() if hasattr(report_instance, "rows") else None + rows = ( + report_instance.rows(current_page_data=None) + if hasattr(report_instance, "rows") + else None + ) writer = csv.writer(response) @@ -122,6 +132,7 @@ def export_report_to_csv(request, report_slug, current_tab=None, current_page=No return response + def export_report_to_excel(request, report_slug): report_class = utils.get_report_by_slug(report_slug) report_instance = report_class() From fc5ea7663b60d46fcd948d01c88852f082b583c0 Mon Sep 17 00:00:00 2001 From: nboyse Date: Thu, 25 Jan 2024 15:52:57 +0000 Subject: [PATCH 04/24] add pandas as a package --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5db4dd0cc..2905d3671 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,6 +47,7 @@ markupsafe==2.1.2 moto==2.1.0 notifications-python-client==6.4.1 openpyxl==3.0.7 +pandas==2.2.0 parsec==3.8 psycopg2-binary==2.9.* py_w3c==0.3.1 From 43af513fc45a14be90a62a3aaf558ae747195eb1 Mon Sep 17 00:00:00 2001 From: nboyse Date: Thu, 25 Jan 2024 15:56:20 +0000 Subject: [PATCH 05/24] add correct version of panads --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2905d3671..bf76e9a43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,7 +47,7 @@ markupsafe==2.1.2 moto==2.1.0 notifications-python-client==6.4.1 openpyxl==3.0.7 -pandas==2.2.0 +pandas==2.0.3 parsec==3.8 psycopg2-binary==2.9.* py_w3c==0.3.1 From 3cea25a4259142f22ff6575e3bb413922566cd2b Mon Sep 17 00:00:00 2001 From: nboyse Date: Tue, 30 Jan 2024 14:25:35 +0000 Subject: [PATCH 06/24] wip: integrate ag grid --- reports/jinja2/generics/table.jinja | 19 +++++++++++++++---- reports/reports/eu_data.py | 10 +++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/reports/jinja2/generics/table.jinja b/reports/jinja2/generics/table.jinja index 87291247b..98ae35473 100644 --- a/reports/jinja2/generics/table.jinja +++ b/reports/jinja2/generics/table.jinja @@ -1,5 +1,18 @@ {% from "components/table/macro.njk" import govukTable -%} + + {% if report.tabular_reports -%}
@@ -62,10 +75,8 @@
{% else -%} - {{ govukTable({ - "head": report.headers(), - "rows": report.rows(current_page_data), - }) }} +
+ {% if current_page_data -%}
{% else -%} -
+
{% if current_page_data -%} @@ -91,7 +95,7 @@ "items" {% endif %}
-
+
Page {{ current_page_data.number }} of {{ current_page_data.paginator.num_pages }}
    diff --git a/reports/jinja2/reports/report_table.jinja b/reports/jinja2/reports/report_table.jinja index 727baea6b..90e104203 100644 --- a/reports/jinja2/reports/report_table.jinja +++ b/reports/jinja2/reports/report_table.jinja @@ -22,7 +22,7 @@

    {{ report.description|safe }}

    -
    +
    {% include "generics/table.jinja" %}
    {% endblock %} diff --git a/reports/reports/eu_data.py b/reports/reports/eu_data.py index 9205b10c9..6ca3a6b60 100644 --- a/reports/reports/eu_data.py +++ b/reports/reports/eu_data.py @@ -32,9 +32,7 @@ class Report(ReportBaseTable): def headers(self) -> [dict]: return [ - {"field": - header.replace("_", " ").capitalize() - } + {"field": header.replace("_", " ").capitalize(), "filter": 'agTextColumnFilter'} for header in self.headers_list ] From ae0b38733f70bee90a1c03b26a7e616250abc46b Mon Sep 17 00:00:00 2001 From: nboyse Date: Tue, 30 Jan 2024 17:47:38 +0000 Subject: [PATCH 08/24] fixing broken reports as a result of this data --- reports/jinja2/generics/table.jinja | 9 ++++++++- reports/reports/eu_data.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/reports/jinja2/generics/table.jinja b/reports/jinja2/generics/table.jinja index 8c9a7687c..306cd7dff 100644 --- a/reports/jinja2/generics/table.jinja +++ b/reports/jinja2/generics/table.jinja @@ -1,4 +1,5 @@ {% from "components/table/macro.njk" import govukTable -%} +{% if current_page_data -%} +{% endif -%} {% if report.tabular_reports -%}
    @@ -79,10 +81,10 @@
    {% else -%} + {% if current_page_data -%}
    - {% if current_page_data -%}
+{% else -%} + {{ govukTable({ + "head": report.headers(), + "rows": report.rows() + }) }} {% endif -%} {% endif -%} \ No newline at end of file diff --git a/reports/reports/eu_data.py b/reports/reports/eu_data.py index 6ca3a6b60..3cb5a5931 100644 --- a/reports/reports/eu_data.py +++ b/reports/reports/eu_data.py @@ -32,12 +32,18 @@ class Report(ReportBaseTable): def headers(self) -> [dict]: return [ - {"field": header.replace("_", " ").capitalize(), "filter": 'agTextColumnFilter'} + { + "field": header.replace("_", " ").capitalize(), + "filter": "agTextColumnFilter", + } for header in self.headers_list ] def row(self, row) -> [dict]: - return {field.replace("_", " ").capitalize(): str(getattr(row, field, None)) for field in self.headers_list} + return { + field.replace("_", " ").capitalize(): str(getattr(row, field, None)) + for field in self.headers_list + } def rows(self, current_page_data) -> [dict]: table_rows = [] From 0150189b63d9dd6402a34aa708c3af6ddc5767f7 Mon Sep 17 00:00:00 2001 From: nboyse Date: Tue, 30 Jan 2024 17:53:29 +0000 Subject: [PATCH 09/24] adjust jinja2 template --- reports/jinja2/generics/table.jinja | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/reports/jinja2/generics/table.jinja b/reports/jinja2/generics/table.jinja index 306cd7dff..d7a5fe81f 100644 --- a/reports/jinja2/generics/table.jinja +++ b/reports/jinja2/generics/table.jinja @@ -1,5 +1,5 @@ {% from "components/table/macro.njk" import govukTable -%} -{% if current_page_data -%} +{% if report.slug() == "table_report_of_eu_data" -%} {% endif -%} {% if report.tabular_reports -%} @@ -84,6 +97,16 @@ {% if report.headers_list -%}
+
+ {% for header in report.headers() %} +
+ + +
+ {% endfor -%} +
{% else -%} {{ govukTable({ "head": report.headers(), From 81153ad8cc1d8a076913ee4b4ec24335628be62b Mon Sep 17 00:00:00 2001 From: nboyse Date: Thu, 15 Feb 2024 16:36:30 +0000 Subject: [PATCH 17/24] resolve inline issue --- reports/jinja2/generics/table.jinja | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/reports/jinja2/generics/table.jinja b/reports/jinja2/generics/table.jinja index ee0cf0cd6..dddd05d1c 100644 --- a/reports/jinja2/generics/table.jinja +++ b/reports/jinja2/generics/table.jinja @@ -29,6 +29,14 @@ }; const grid = new agGrid.createGrid(document.querySelector('#myGrid'), gridOptions); + var checkboxes = document.querySelectorAll('.govuk-checkboxes__input'); + + checkboxes.forEach(function (checkbox) { + checkbox.addEventListener('change', function () { + var fieldName = this.id.replace('toggleFor', ''); + toggleColumnVisibility(fieldName); + }); + }); }); {% endif -%} @@ -100,7 +108,7 @@
{% for header in report.headers() %}
- + From 740623b03c8c059fe47020ab5970bb80f5a80534 Mon Sep 17 00:00:00 2001 From: nboyse Date: Mon, 19 Feb 2024 14:45:02 +0000 Subject: [PATCH 18/24] finally got icons working --- reports/jinja2/generics/table.jinja | 2 +- settings/common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/reports/jinja2/generics/table.jinja b/reports/jinja2/generics/table.jinja index dddd05d1c..333781c90 100644 --- a/reports/jinja2/generics/table.jinja +++ b/reports/jinja2/generics/table.jinja @@ -1,5 +1,5 @@ {% from "components/table/macro.njk" import govukTable -%} -{% if report.headers_list -%} +{% if report.headers_list -%} {% endif -%} From 245dd75bbf0c10a943f5485e4ba15474785e4d02 Mon Sep 17 00:00:00 2001 From: nboyse Date: Thu, 22 Feb 2024 13:15:24 +0000 Subject: [PATCH 23/24] feat: change ordering, format fields correctly etc --- reports/reports/eu_data.py | 34 ++++++++++++++++++++++-------- reports/tests/test_report_views.py | 13 ++++++++++++ reports/views.py | 14 ++++++++++-- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/reports/reports/eu_data.py b/reports/reports/eu_data.py index 0e241110b..24e2b2fd8 100644 --- a/reports/reports/eu_data.py +++ b/reports/reports/eu_data.py @@ -15,7 +15,6 @@ class Report(ReportBaseTable): "order_no", "start_date", "end_date", - "red_ind", "origin", "measure_type", "legal_base", @@ -28,22 +27,39 @@ class Report(ReportBaseTable): "measure_exists", ] - headers_list = sorted(headers_list) - def headers(self) -> List[Dict[str, Union[str, str]]]: return [ { "field": header.replace("_", " ").capitalize(), - "filter": "agTextColumnFilter", + "filter": self.get_filter(header), } for header in self.headers_list ] + + def get_filter(self, header): + if "date" in header: + return "agDateColumnFilter" + else: + return "agTextColumnFilter" def row(self, row) -> Dict[str, Union[str, Optional[str]]]: - return { - field.replace("_", " ").capitalize(): str(getattr(row, field, None)) - for field in self.headers_list - } + updated_row = {} + for field in self.headers_list: + value = getattr(row, field, None) + display_value = self.format_field(field, value) + updated_row[field.replace("_", " ").capitalize()] = display_value + return updated_row + + def format_field(self, field: str, value: Optional[str]) -> str: + if "exists" in field: + if value == "UNKNOWN": + return "?" + elif value == "EXISTS IN TAP": + return "Y" + else: + return "N" + else: + return "" if value is None else str(value) def rows(self) -> List[Dict[str, Union[str, Optional[str]]]]: table_rows = [] @@ -56,4 +72,4 @@ def query(self): try: return EUDataModel.objects.all().order_by("goods_code") except EUDataModel.DoesNotExist: - return None + return None \ No newline at end of file diff --git a/reports/tests/test_report_views.py b/reports/tests/test_report_views.py index bbbabb3f1..26f5e3b7e 100644 --- a/reports/tests/test_report_views.py +++ b/reports/tests/test_report_views.py @@ -3,10 +3,13 @@ from django.urls import reverse from django.test import RequestFactory +from common.models.user import User + from reports.utils import get_reports from reports.views import export_report_to_csv, export_report_to_excel from reports.reports.expiring_quotas_with_no_definition_period import Report from reports.reports.cds_approved import Report as ChartReport +from reports.forms import UploadCSVForm pytestmark = pytest.mark.django_db @@ -77,3 +80,13 @@ def test_export_report_to_excel(self, request): response["Content-Disposition"] == f'attachment; filename="{report_slug}_report.xlsx"' ) + + + def test_upload_report_csv_get(self, request): + client = request.getfixturevalue("valid_user_client") + + response = client.get(reverse("reports:upload_report_csv")) + + assert response.status_code == 200 + + assert 'Upload CSV' in response.content.decode("utf-8") \ No newline at end of file diff --git a/reports/views.py b/reports/views.py index 77cf51d0c..20ee8dfd9 100644 --- a/reports/views.py +++ b/reports/views.py @@ -237,10 +237,20 @@ def save_eu_data(content): else None ) + order_no = row.get("Order No.", None) + if order_no is not None and order_no != "": + order_no = str(int(float(order_no))).zfill(6) + else: + order_no = "" + + goods_code = row.get("Goods code", None) + if goods_code is not None and goods_code != "": + goods_code = str(int(float(goods_code))).zfill(10) + instance = EUDataModel( - goods_code=row.get("Goods code", None), + goods_code=goods_code, add_code=row.get("Add code", None), - order_no=row.get("Order No.", None), + order_no=order_no, start_date=start_date, end_date=end_date, red_ind=row.get("RED_IND", None), From febbbd4c01563a4d2e54f1d835f3f498cd32d33a Mon Sep 17 00:00:00 2001 From: nboyse Date: Thu, 22 Feb 2024 14:45:13 +0000 Subject: [PATCH 24/24] reordering and renaming of columns --- reports/migrations/0005_auto_20240222_1442.py | 23 +++++++++++++++++++ reports/models.py | 4 ++-- reports/reports/eu_data.py | 12 +++++----- reports/views.py | 4 ++-- 4 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 reports/migrations/0005_auto_20240222_1442.py diff --git a/reports/migrations/0005_auto_20240222_1442.py b/reports/migrations/0005_auto_20240222_1442.py new file mode 100644 index 000000000..a93eb2305 --- /dev/null +++ b/reports/migrations/0005_auto_20240222_1442.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.24 on 2024-02-22 14:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0004_auto_20240125_1134'), + ] + + operations = [ + migrations.RenameField( + model_name='eudatamodel', + old_name='origin', + new_name='geographical_area_origin', + ), + migrations.RenameField( + model_name='eudatamodel', + old_name='goods_code', + new_name='goods_nomenclature_code', + ), + ] diff --git a/reports/models.py b/reports/models.py index 9ada0b99d..12d1565ca 100644 --- a/reports/models.py +++ b/reports/models.py @@ -7,13 +7,13 @@ class Meta: class EUDataModel(models.Model): - goods_code = models.CharField(max_length=255, null=True, blank=True) + goods_nomenclature_code = models.CharField(max_length=255, null=True, blank=True) add_code = models.CharField(max_length=255, null=True, blank=True) order_no = models.CharField(max_length=255, null=True, blank=True) start_date = models.DateField(null=True, blank=True) end_date = models.DateField(null=True, blank=True) red_ind = models.CharField(max_length=255, null=True, blank=True) - origin = models.CharField(max_length=255, null=True, blank=True) + geographical_area_origin = models.CharField(max_length=255, null=True, blank=True) measure_type = models.CharField(max_length=255, null=True, blank=True) legal_base = models.CharField(max_length=255, null=True, blank=True) duty = models.CharField(max_length=255, null=True, blank=True) diff --git a/reports/reports/eu_data.py b/reports/reports/eu_data.py index 24e2b2fd8..36980ef58 100644 --- a/reports/reports/eu_data.py +++ b/reports/reports/eu_data.py @@ -10,20 +10,20 @@ class Report(ReportBaseTable): enabled = True headers_list = [ - "goods_code", + "goods_nomenclature_exists", + "goods_nomenclature_code", "add_code", "order_no", "start_date", "end_date", - "origin", + "geographical_area_exists", + "geographical_area_origin", + "measure_type_exists", "measure_type", "legal_base", "duty", "origin_code", "meas_type_code", - "goods_nomenclature_exists", - "geographical_area_exists", - "measure_type_exists", "measure_exists", ] @@ -70,6 +70,6 @@ def rows(self) -> List[Dict[str, Union[str, Optional[str]]]]: def query(self): try: - return EUDataModel.objects.all().order_by("goods_code") + return EUDataModel.objects.all().order_by("goods_nomenclature_code") except EUDataModel.DoesNotExist: return None \ No newline at end of file diff --git a/reports/views.py b/reports/views.py index 20ee8dfd9..2871e02ff 100644 --- a/reports/views.py +++ b/reports/views.py @@ -248,13 +248,13 @@ def save_eu_data(content): goods_code = str(int(float(goods_code))).zfill(10) instance = EUDataModel( - goods_code=goods_code, + goods_nomenclature_code=goods_code, add_code=row.get("Add code", None), order_no=order_no, start_date=start_date, end_date=end_date, red_ind=row.get("RED_IND", None), - origin=row.get("Origin", None), + geographical_area_origin=row.get("Origin", None), measure_type=row.get(" Measure type", None), legal_base=row.get("Legal base", None), duty=row.get("Duty", None),