Migrations to TM 4

.dependabot/config.yml

@@ -0,0 +1,10 @@
+version: 1
+
+update_configs:
+  - package_manager: "javascript"
+    directory: "/frontend"
+    update_schedule: "live"
+
+  - package_manager: "python"
+    directory: "/"
+    update_schedule: "live"

.dockerignore

@@ -0,0 +1,7 @@
+.*
+*.yml
+**/*.pyc
+**/node_modules
+**/npm-debug.log
+logs/
+docs/

.flake8

@@ -1,2 +1,2 @@
 [flake8]
-max-line-length = 120
+max-line-length = 119

.github/labeler.yml

@@ -0,0 +1,15 @@
+"Component: Frontend":
+  - frontend/**/*
+"Component: Backend":
+  - backend/**/*
+  - tests/**/*
+  - migrations/**/*
+  - ./manage.py
+  - ./requirements.txt
+"Component: Infrastructure":
+  - .circleci/*
+  - .github/**/*
+  - scripts/aws/**/*
+  - scripts/docker/**/*
+  - scripts/install/**/*
+  - docker-*.yml

.github/workflows/label.yml

@@ -0,0 +1,20 @@
+# This workflow will triage pull requests and apply a label based on the
+# paths that are modified in the pull request.
+#
+# To use this workflow, you will need to set up a .github/labeler.yml
+# file with configuration.  For more information, see:
+# https://github.com/actions/labeler
+
+name: Labeler
+on:
+- pull_request_target
+
+jobs:
+  label:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/labeler@v2
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.gitignore

@@ -3,24 +3,25 @@
 tasking-manager*.env
 
 # Generated frontend #
-client/node_modules/
-client/assets/styles/css
-client/app/taskingmanager.config.js
-client/id_preset_categories.json
-client/app/id_preset_categories.js
-client/package-lock.json
-client/yarn.lock
+frontend/node_modules/
+frontend/build/
+frontend/public/static/
+frontend/assets/styles/
+frontend/package-lock.json
+frontend/.env
+frontend/.eslintcache
+frontend/coverage/
 
 # Ignore dist folder as this contains the minimized build for deploy
-server/web/static/dist
-server/web/static/dist/
+backend/web/static/dist
+backend/web/static/dist/
 
 # Ignore local logs
 logs/
 *.log
 
 # Ignore some translation stuff
-.tx/tasking-manager.master/
+.tx/
 
 #python
 venv/

.gitlab-ci.yml

@@ -0,0 +1,169 @@
+variables:
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+default:
+  image: registry.gitlab.com/gokaart/tasking-manager/ci:latest
+  before_script:
+    - |
+      echo "Install requirements"
+      # Install NPM packages and build frontend
+      cd frontend
+      CI=true yarn
+      cd ..
+      # Install Python dependencies
+      python3 -m venv env
+      source env/bin/activate
+      pip install --upgrade pip
+      pip install -r requirements.txt
+      # Install python gdal -- it MUST match the version of gdal installed
+      pip install gdal==$(gdalinfo --version | awk '{print $2}' | awk -F, '{print $1}')
+  cache:
+    key: "cached-dependencies"
+    policy: pull
+    paths:
+      - .cache/npm
+      - .cache/pip
+      - frontend/node_modules
+      - env
+
+image_updates:
+  image: docker:20.10.3
+  stage: build
+  services:
+    - docker:20.10.3-dind
+  before_script:
+    - docker info
+    - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
+  tags:
+    - docker
+  script:
+    - docker pull ${CI_REGISTRY_IMAGE}/tasking-manager:frontend || true
+    - docker pull ${CI_REGISTRY_IMAGE}/tasking-manager:backend || true
+    - echo "TM_APP_BASE_URL=\"${TM_APP_BASE_URL}\"" > tasking-manager.env
+    - echo "TM_APP_API_URL=\"${TM_APP_API_URL}\"" >> tasking-manager.env
+    - echo "TM_APP_API_VERSION=\"${TM_APP_API_VERSION}\"" >> tasking-manager.env
+    - docker build --build-arg branch=${CI_COMMIT_REF_NAME} --cache-from ${CI_REGISTRY_IMAGE}/tasking-manager:frontend -t ${CI_REGISTRY_IMAGE}/tasking-manager:frontend-${CI_COMMIT_REF_NAME} --file scripts/docker/Dockerfile.frontend .
+    - docker build --build-arg branch=${CI_COMMIT_REF_NAME} --cache-from ${CI_REGISTRY_IMAGE}/tasking-manager:backend -t ${CI_REGISTRY_IMAGE}/tasking-manager:backend-${CI_COMMIT_REF_NAME} --file scripts/docker/Dockerfile.backend .
+    - |
+      if [[ ! -z ${CI_REGISTRY+x} ]]; then
+        docker push ${CI_REGISTRY_IMAGE}/tasking-manager:backend-${CI_COMMIT_REF_NAME}
+        docker push ${CI_REGISTRY_IMAGE}/tasking-manager:frontend-${CI_COMMIT_REF_NAME}
+      fi
+    - |
+      if [[ ${CI_COMMIT_REF_PROTECTED} ]]; then
+        docker pull ${CI_REGISTRY_IMAGE}/ci || true
+        docker build --cache-from ${CI_REGISTRY_IMAGE}/ci -t ${CI_REGISTRY_IMAGE}/ci scripts/docker/ci
+        docker push ${CI_REGISTRY_IMAGE}/ci:latest
+        docker tag ${CI_REGISTRY_IMAGE}/tasking-manager:backend-${CI_COMMIT_REF_NAME} ${CI_REGISTRY_IMAGE}/tasking-manager:backend
+        docker push ${CI_REGISTRY_IMAGE}/tasking-manager:backend
+        docker tag ${CI_REGISTRY_IMAGE}/tasking-manager:frontend-${CI_COMMIT_REF_NAME} ${CI_REGISTRY_IMAGE}/tasking-manager:frontend
+        docker push ${CI_REGISTRY_IMAGE}/tasking-manager:frontend
+      fi
+  interruptible: true
+
+
+
+install requirements:
+  image: registry.gitlab.com/gokaart/tasking-manager/ci:latest
+  stage: build
+  cache:
+    key: "cached-dependencies"
+    policy: pull-push
+    paths:
+      - .cache/npm
+      - .cache/pip
+      - frontend/node_modules
+      - env
+  script:
+    - echo "Just a job to pre-cache dependencies"
+  interruptible: true
+
+javascript tests:
+  image: registry.gitlab.com/gokaart/tasking-manager/ci:latest
+  stage: test
+  script:
+    - |
+      echo "Run JS Unit tests"
+      # JS Unit Tests
+      cd frontend
+      yarn add --dev jest-junit
+      CI=true yarn test --coverage --coverageReporters=html --coverageReporters=cobertura --reporters=default --reporters=jest-junit
+      CI=true GENERATE_SOURCEMAP=false yarn build
+  interruptible: true
+  needs: ["install requirements"]
+  artifacts:
+    paths:
+      - frontend/junit.xml
+      - frontend/coverage
+    reports:
+      junit:
+        - frontend/junit.xml
+      cobertura:
+        - frontend/coverage/cobertura-coverage.xml
+
+python lint:
+  image: registry.gitlab.com/gokaart/tasking-manager/ci:latest
+  stage: test
+  script: 
+    - |
+      mkdir tests/backend/lint
+      . env/bin/activate
+      black --check manage.py backend tests migrations
+      flake8 manage.py backend tests migrations
+  interruptible: true
+  needs: ["install requirements"]
+
+python tests:
+  image: registry.gitlab.com/gokaart/tasking-manager/ci:latest
+  stage: test
+  services:
+    - name: registry.gitlab.com/gokaart/docker-images/postgis:latest
+  variables:
+    APP_BASE_URL: "http://127.0.0.1"
+    POSTGRES_DB: test_database
+    POSTGRES_USER: runner
+    POSTGRES_PASSWORD: "completely random password here!"
+    POSTGRES_ENDPOINT: test_database
+  script:
+    - source env/bin/activate
+    - echo "APP_BASE_URL=\"${APP_BASE_URL}\"" > backend/tasking-manager.env
+    - echo "POSTGRES_DB=\"${POSTGRES_DB}\"" >> backend/tasking-manager.env
+    - echo "POSTGRES_USER=\"${POSTGRES_USER}\"" >> backend/tasking-manager.env
+    - echo "POSTGRES_PASSWORD=\"${POSTGRES_PASSWORD}\"" >> backend/tasking-manager.env
+    - echo "POSTGRES_ENDPOINT=\"${POSTGRES_ENDPOINT}\"" >> backend/tasking-manager.env
+    - cat backend/tasking-manager.env
+    - |
+      echo "Run Python Tests"
+      # Run Python tests
+      python3 -m unittest discover tests/backend
+      mkdir -p tests/server/results
+      find ./tests/server -name "test*.py" -exec chmod -x {} \;
+      ./env/bin/nosetests ./tests/server --with-xunit \
+        --xunit-file $CI_PROJECT_DIR/tests/server/results/unitresults.xml \
+        --with-coverage --cover-erase --cover-package=./server
+      ./env/bin/coverage xml -o $CI_PROJECT_DIR/tests/server/results/coverage.xml
+  interruptible: true
+  needs: ["install requirements"]
+  artifacts:
+    paths:
+      - test/server/results/
+    reports:
+      junit:
+        - tests/server/results/unitresults.xml
+      cobertura:
+        - tests/server/results/coverage.xml
+
+code navigation:
+  image: registry.gitlab.com/gokaart/tasking-manager/ci:latest
+  stage: deploy
+  script:
+    - |
+      echo "Building code navigation"
+      npm install @sourcegraph/lsif-tsc
+      npx lsif-tsc frontend/src/**/* --AllowJs --checkJs --out dump_frontend.lsif
+  interruptible: true
+  needs: ["install requirements"]
+  artifacts:
+    reports:
+      lsif:
+        - dump_frontend.lsif

.tx/config

@@ -1,37 +1,31 @@
 [main]
 host = https://www.transifex.com
 
-[tasking-manager.master]
-source_file = client/locale/en.json
+[tasking-manager.version-4]
+source_file = frontend/src/locales/en.json
 source_lang = en
-trans.ar = client/locale/ar.json
-trans.bn = client/locale/bn.json
-trans.cs = client/locale/cs.json
-trans.da = client/locale/da.json
-trans.de = client/locale/de.json
-trans.es = client/locale/es.json
-trans.fa_IR = client/locale/fa_IR.json
-trans.fi = client/locale/fi.json
-trans.fr = client/locale/fr.json
-trans.gl = client/locale/gl.json
-trans.hu = client/locale/hu.json
-trans.id = client/locale/id.json
-trans.it = client/locale/it.json
-trans.ja = client/locale/ja.json
-trans.ko = client/locale/ko.json
-trans.lt = client/locale/lt.json
-trans.mg = client/locale/mg.json
-trans.nb = client/locale/nb.json
-trans.nl_NL = client/locale/nl_NL.json
-trans.pl = client/locale/pl.json
-trans.pt = client/locale/pt.json
-trans.pt_BR = client/locale/pt_BR.json
-trans.ru = client/locale/ru.json
-trans.si = client/locale/si.json
-trans.sl = client/locale/sl.json
-trans.ta = client/locale/ta.json
-trans.uk = client/locale/uk.json
-trans.vi = client/locale/vi.json
-trans.zh_TW = client/locale/zh_TW.json
+trans.ar = frontend/src/locales/ar.json
+trans.cs = frontend/src/locales/cs.json
+trans.de = frontend/src/locales/de.json
+trans.el = frontend/src/locales/el.json
+trans.es = frontend/src/locales/es.json
+trans.fa_IR = frontend/src/locales/fa_IR.json
+trans.fr = frontend/src/locales/fr.json
+trans.he = frontend/src/locales/he.json
+trans.hu = frontend/src/locales/hu.json
+trans.id = frontend/src/locales/id.json
+trans.it = frontend/src/locales/it.json
+trans.ja = frontend/src/locales/ja.json
+trans.ko = frontend/src/locales/ko.json
+trans.mg = frontend/src/locales/mg.json
+trans.ml = frontend/src/locales/ml.json
+trans.nl_NL = frontend/src/locales/nl_NL.json
+trans.pt = frontend/src/locales/pt.json
+trans.pt_BR = frontend/src/locales/pt_BR.json
+trans.sv = frontend/src/locales/sv.json
+trans.sw = frontend/src/locales/sw.json
+trans.tl = frontend/src/locales/tl.json
+trans.tr = frontend/src/locales/tr.json
+trans.uk = frontend/src/locales/uk.json
+trans.zh_TW = frontend/src/locales/zh_TW.json
 type = KEYVALUEJSON
-

Dockerfile

@@ -0,0 +1,48 @@
+FROM quay.io/hotosm/base-python-image as base
+LABEL version=0.1
+LABEL maintainer="HOT Sysadmin <[email protected]>"
+LABEL description="Builds backend docker image"
+
+WORKDIR /usr/src/app
+
+FROM base as builder
+
+# Setup backend build-time dependencies
+RUN apk update && \
+    apk add \
+        postgresql-dev \
+        gcc \
+        g++ \
+        python3-dev \
+        musl-dev \
+        libffi-dev \
+        geos-dev \
+        proj-util \
+        proj-dev \
+        make
+
+COPY requirements.txt .
+
+RUN pip install \
+    --prefix=/install \
+    --no-cache-dir \
+    --no-warn-script-location \
+    -r requirements.txt
+
+# Setup backend runtime dependencies
+FROM base
+
+RUN apk update && \
+    apk add \
+        postgresql-libs geos proj-util
+
+COPY --from=builder /install /usr/local
+COPY backend backend/
+COPY migrations migrations/
+COPY scripts/world scripts/world/
+COPY scripts/database scripts/database/
+COPY manage.py .
+
+ENV TZ UTC # Fix timezone (do not change - see issue #3638)
+EXPOSE 5000
+CMD ["gunicorn", "-c", "python:backend.gunicorn", "manage:application"]