Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(iatlas): update iatlas-data with the latest improvements made to GitLab (ARCH-315, ARCH-317) #2900

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions apps/iatlas/data/.dockerignore

This file was deleted.

11 changes: 11 additions & 0 deletions apps/iatlas/data/.flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
[flake8]
exclude =
.git,
__pycache__,
build,
dist,
.tox,
venv,
.venv,
.pytest_cache
max-line-length = 120
2 changes: 1 addition & 1 deletion apps/iatlas/data/.python-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.10.13
3.11.10
69 changes: 57 additions & 12 deletions apps/iatlas/data/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,33 +1,78 @@
FROM python:3.10.13
FROM python:3.11.10

ARG USERNAME=app
ARG USER_UID=1000
ARG USER_GID=$USER_UID
ENV APP_DIR=/opt/app \
APP_USERNAME=${USERNAME}

SHELL ["/bin/bash", "-o", "pipefail", "-c"]

RUN apt-get update -qq -y && export DEBIAN_FRONTEND=noninteractive \
RUN groupadd --gid "$USER_GID" "$USERNAME" \
&& useradd --uid "$USER_UID" --gid "$USER_GID" -m "$USERNAME" \
# Install sudo
&& apt-get update -qq -y && export DEBIAN_FRONTEND=noninteractive \
&& apt-get install --no-install-recommends -qq -y \
wget \
gosu wget \
# Install the GPG key for the Postgres repo
&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
# Add the repo
&& echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \
&& apt-get update -qq -y \
# Install the Postgres 12 client
# Install the Postgres 15 client
&& apt-get install --no-install-recommends -qq -y \
postgresql-client-12 \
&& mkdir -p /home/gitlab/sage-iatlas-data \
postgresql-client-15 \
&& apt-get -y autoclean \
&& apt-get -y autoremove \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /
COPY requirements.txt requirements.txt
COPY src/ /src
WORKDIR ${APP_DIR}
COPY src src/
COPY pyproject.toml poetry.lock ./

RUN pip install --no-cache-dir -r requirements.txt
# Use the version of Poetry installed in the dev container.
# See /workspaces/sage-monorepo/tools/devcontainers/sage/.devcontainer/Dockerfile
RUN pip install --no-cache-dir poetry==1.8.3 \
&& poetry config --local virtualenvs.create false \
&& poetry install --no-root --no-interaction --no-ansi \
&& pip cache purge

WORKDIR /
COPY docker-entrypoint.sh ./
RUN chmod +x docker-entrypoint.sh

ENTRYPOINT ["/docker-entrypoint.sh"]

# Set up the iAtlas database
CMD ["python", "/src/build_database.py"]
CMD ["python", "src/build_database.py"]



# RUN apt-get update -qq -y && export DEBIAN_FRONTEND=noninteractive \
# && apt-get install --no-install-recommends -qq -y \
# wget \
# # Install the GPG key for the Postgres repo
# && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
# # Add the repo
# && echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \
# && apt-get update -qq -y \
# # Install the Postgres 15 client
# && apt-get install --no-install-recommends -qq -y \
# postgresql-client-15 \
# && mkdir -p /home/gitlab/sage-iatlas-data \
# && apt-get -y autoclean \
# && apt-get -y autoremove \
# && rm -rf /var/lib/apt/lists/*

# WORKDIR /
# COPY requirements.txt requirements.txt
# COPY src/ /src

# RUN pip install --no-cache-dir -r requirements.txt

# COPY docker-entrypoint.sh ./
# RUN chmod +x docker-entrypoint.sh

# ENTRYPOINT ["/docker-entrypoint.sh"]

# # Set up the iAtlas database
# CMD ["python", "/src/build_database.py"]
6 changes: 6 additions & 0 deletions apps/iatlas/data/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,9 @@

Creates a Docker container that download iAtlas data and data schema from Synapse before seeding the
iAtlas DB.

## Prepare the project

```console
nx prepare iatlas-data
```
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env bash

PYTHON_VERSION="3.10.13"
PYTHON_VERSION="3.11.10"

pyenv install --skip-existing $PYTHON_VERSION

Expand All @@ -9,6 +9,5 @@ pyenv install --skip-existing $PYTHON_VERSION
eval "$(pyenv init -)"

pyenv local $PYTHON_VERSION
poetry env use $PYTHON_VERSION
poetry install
# poetry install --with prod,dev
poetry env use $(pyenv which python)
poetry install --with dev
2,388 changes: 1,381 additions & 1,007 deletions apps/iatlas/data/poetry.lock

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion apps/iatlas/data/poetry.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
[virtualenvs]
in-project = true
create = true
66 changes: 34 additions & 32 deletions apps/iatlas/data/project.json
Original file line number Diff line number Diff line change
@@ -1,35 +1,52 @@
{
"name": "iatlas-data",
"$schema": "../../../node_modules/nx/schemas/project-schema.json",
"sourceRoot": "apps/iatlas/data/scripts",
"projectType": "application",
"sourceRoot": "apps/iatlas/data/src",
"targets": {
"create-config": {
"lock": {
"executor": "nx:run-commands",
"options": {
"command": "cp -n .env.example .env",
"command": "poetry lock --no-update",
"cwd": "{projectRoot}"
}
},
"build": {
"executor": "@nxlv/python:build",
"outputs": ["{projectRoot}/dist"],
"options": {
"outputPath": "{projectRoot}/dist",
"publish": false,
"lockedVersions": true,
"bundleLocalDependencies": true
}
},
"prepare": {
"executor": "nx:run-commands",
"options": {
"command": "./prepare-python.sh",
"command": "./install.sh",
"cwd": "{projectRoot}"
}
},
"serve": {
"update": {
"executor": "nx:run-commands",
"options": {
"command": "docker compose up",
"cwd": "apps/iatlas/data"
},
"dependsOn": []
"command": "poetry update",
"cwd": "{projectRoot}"
}
},
"lint": {
"executor": "@nxlv/python:flake8",
"outputs": ["{workspaceRoot}/reports/{projectRoot}/pylint.txt"],
"options": {
"outputFile": "reports/{projectRoot}/pylint.txt"
}
},
"serve-detach": {
"serve": {
"executor": "nx:run-commands",
"options": {
"command": "docker/iatlas/serve-detach.sh {projectName}"
"command": "poetry run python src/build_database.py",
"cwd": "{projectRoot}"
}
},
"build-image": {
Expand All @@ -42,27 +59,12 @@
},
"push": false
}
},
"publish-image": {
"executor": "@nx-tools/nx-container:build",
"options": {
"context": "apps/iatlas/data",
"metadata": {
"images": ["ghcr.io/sage-bionetworks/{projectName}"],
"tags": ["type=edge,branch=main", "type=sha"]
},
"push": true
},
"dependsOn": ["build-image"]
},
"scan-image": {
"executor": "nx:run-commands",
"options": {
"command": "trivy image ghcr.io/sage-bionetworks/{projectName}:local --quiet",
"color": true
}
}
},
"tags": ["type:db", "scope:backend", "language:python"],
"implicitDependencies": []
"tags": ["language:python", "package-manager:poetry"],
"release": {
"version": {
"generator": "@nxlv/python:release-version"
}
}
}
36 changes: 29 additions & 7 deletions apps/iatlas/data/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,15 +1,37 @@
[tool.coverage.run]
branch = true
source = [ "src" ]

[tool.coverage.report]
exclude_lines = [ "if TYPE_CHECKING:" ]
show_missing = true

[tool.pytest.ini_options]
addopts = "--cov --cov-report html:'../../../coverage/apps/iatlas/data/html' --cov-report xml:'../../../coverage/apps/iatlas/data/coverage.xml' --html='../../../reports/apps/iatlas/data/unittests/html/index.html' --junitxml='../../../reports/apps/iatlas/data/unittests/junit.xml'"

[tool.poetry]
name = "iatlas-data"
version = "0.1.0"
description = ""
authors = ["andrewelamb <[email protected]>"]
version = "1.0.0"
description = "Automatically generated by Nx."
authors = [ ]
license = "Proprietary"
readme = "README.md"

[tool.poetry.dependencies]
python = "^3.9"
schematic-db = {version = "0.0.36", extras = ["postgres", "synapse"]}
[[tool.poetry.packages]]
include = "src"

[tool.poetry.dependencies]
python = "3.11.10"
schematic-db = {version = "0.1.5", extras = ["postgres", "synapse"]}

[tool.poetry.group.dev.dependencies]
autopep8 = "2.0.2"
flake8 = "6.0.0"
pytest = "7.3.1"
pytest-sugar = "0.9.7"
pytest-cov = "4.1.0"
pytest-html = "3.2.0"

[build-system]
requires = ["poetry-core"]
requires = [ "poetry-core" ]
build-backend = "poetry.core.masonry.api"
7 changes: 7 additions & 0 deletions apps/iatlas/data/pyrightconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"include": ["src"],
"exclude": ["tests"],
"venvPath": ".",
"venv": ".venv",
"pythonVersion": "3.11"
}
98 changes: 51 additions & 47 deletions apps/iatlas/data/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,47 +1,51 @@
attrs==23.2.0 ; python_version >= "3.9" and python_version < "4.0"
backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0"
certifi==2024.2.2 ; python_version >= "3.9" and python_version < "4.0"
charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "4.0"
click==8.1.7 ; python_version >= "3.9" and python_version < "4.0"
colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
decorator==5.1.1 ; python_version >= "3.9" and python_version < "4.0"
deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0"
deprecation==2.1.0 ; python_version >= "3.9" and python_version < "4.0"
googleapis-common-protos==1.62.0 ; python_version >= "3.9" and python_version < "4.0"
greenlet==3.0.3 ; python_version >= "3.9" and python_version < "4.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32")
idna==3.6 ; python_version >= "3.9" and python_version < "4.0"
importlib-metadata==6.11.0 ; python_version >= "3.9" and python_version < "4.0"
interrogate==1.5.0 ; python_version >= "3.9" and python_version < "4.0"
nest-asyncio==1.6.0 ; python_version >= "3.9" and python_version < "4.0"
networkx==2.8.8 ; python_version >= "3.9" and python_version < "4.0"
numpy==1.26.4 ; python_version >= "3.9" and python_version <= "3.11" or python_version >= "3.12" and python_version < "4.0"
opentelemetry-api==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
opentelemetry-exporter-otlp-proto-common==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
opentelemetry-exporter-otlp-proto-http==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
opentelemetry-proto==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
opentelemetry-sdk==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
opentelemetry-semantic-conventions==0.42b0 ; python_version >= "3.9" and python_version < "4.0"
packaging==23.2 ; python_version >= "3.9" and python_version < "4.0"
pandas==2.2.1 ; python_version >= "3.9" and python_version < "4.0"
protobuf==4.25.3 ; python_version >= "3.9" and python_version < "4.0"
psycopg2-binary==2.9.9 ; python_version >= "3.9" and python_version < "4.0"
py==1.11.0 ; python_version >= "3.9" and python_version < "4.0"
pydantic==1.10.14 ; python_version >= "3.9" and python_version < "4.0"
python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
pytz==2024.1 ; python_version >= "3.9" and python_version < "4.0"
pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "4.0"
requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0"
schematic-db[postgres,synapse]==0.0.41 ; python_version >= "3.9" and python_version < "4.0"
six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
sqlalchemy-utils==0.41.1 ; python_version >= "3.9" and python_version < "4.0"
sqlalchemy==2.0.27 ; python_version >= "3.9" and python_version < "4.0"
synapseclient==4.1.1 ; python_version >= "3.9" and python_version < "4.0"
tabulate==0.9.0 ; python_version >= "3.9" and python_version < "4.0"
tenacity==8.2.3 ; python_version >= "3.9" and python_version < "4.0"
toml==0.10.2 ; python_version >= "3.9" and python_version < "4.0"
typing-extensions==4.10.0 ; python_version >= "3.9" and python_version < "4.0"
tzdata==2024.1 ; python_version >= "3.9" and python_version < "4.0"
urllib3==1.26.18 ; python_version >= "3.9" and python_version < "4.0"
validators==0.20.0 ; python_version >= "3.9" and python_version < "4.0"
wrapt==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
zipp==3.17.0 ; python_version >= "3.9" and python_version < "4.0"
annotated-types==0.7.0 ; python_version >= "3.11" and python_version < "4.0"
anyio==4.4.0 ; python_version >= "3.11" and python_version < "4.0"
async-lru==2.0.4 ; python_version >= "3.11" and python_version < "4.0"
asyncio-atexit==1.0.1 ; python_version >= "3.11" and python_version < "4.0"
backoff==2.2.1 ; python_version >= "3.11" and python_version < "4.0"
certifi==2024.7.4 ; python_version >= "3.11" and python_version < "4.0"
charset-normalizer==3.3.2 ; python_version >= "3.11" and python_version < "4.0"
colorama==0.4.6 ; python_version >= "3.11" and python_version < "4.0" and platform_system == "Windows"
deprecated==1.2.14 ; python_version >= "3.11" and python_version < "4.0"
deprecation==2.1.0 ; python_version >= "3.11" and python_version < "4.0"
googleapis-common-protos==1.63.2 ; python_version >= "3.11" and python_version < "4.0"
greenlet==3.0.3 ; python_version < "3.13" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version >= "3.11"
h11==0.14.0 ; python_version >= "3.11" and python_version < "4.0"
httpcore==1.0.5 ; python_version >= "3.11" and python_version < "4.0"
httpx==0.27.0 ; python_version >= "3.11" and python_version < "4.0"
idna==3.7 ; python_version >= "3.11" and python_version < "4.0"
importlib-metadata==6.11.0 ; python_version >= "3.11" and python_version < "4.0"
nest-asyncio==1.6.0 ; python_version >= "3.11" and python_version < "4.0"
networkx==3.3 ; python_version >= "3.11" and python_version < "4.0"
numpy==2.0.1 ; python_version >= "3.12" and python_version < "4.0" or python_version == "3.11"
opentelemetry-api==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
opentelemetry-exporter-otlp-proto-common==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
opentelemetry-exporter-otlp-proto-http==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
opentelemetry-proto==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
opentelemetry-sdk==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
opentelemetry-semantic-conventions==0.42b0 ; python_version >= "3.11" and python_version < "4.0"
packaging==24.1 ; python_version >= "3.11" and python_version < "4.0"
pandas==2.2.2 ; python_version >= "3.11" and python_version < "4.0"
protobuf==4.25.4 ; python_version >= "3.11" and python_version < "4.0"
psutil==5.9.8 ; python_version >= "3.11" and python_version < "4.0"
psycopg2-binary==2.9.9 ; python_version >= "3.11" and python_version < "4.0"
pydantic-core==2.20.1 ; python_version >= "3.11" and python_version < "4.0"
pydantic==2.8.2 ; python_version >= "3.11" and python_version < "4.0"
python-dateutil==2.9.0.post0 ; python_version >= "3.11" and python_version < "4.0"
pytz==2024.1 ; python_version >= "3.11" and python_version < "4.0"
pyyaml==6.0.1 ; python_version >= "3.11" and python_version < "4.0"
requests==2.32.3 ; python_version >= "3.11" and python_version < "4.0"
schematic-db[postgres,synapse]==0.1.5 ; python_version >= "3.11" and python_version < "4.0"
six==1.16.0 ; python_version >= "3.11" and python_version < "4.0"
sniffio==1.3.1 ; python_version >= "3.11" and python_version < "4.0"
sqlalchemy-utils==0.41.2 ; python_version >= "3.11" and python_version < "4.0"
sqlalchemy==2.0.31 ; python_version >= "3.11" and python_version < "4.0"
synapseclient==4.4.0 ; python_version >= "3.11" and python_version < "4.0"
tenacity==9.0.0 ; python_version >= "3.11" and python_version < "4.0"
tqdm==4.66.4 ; python_version >= "3.11" and python_version < "4.0"
typing-extensions==4.12.2 ; python_version >= "3.11" and python_version < "4.0"
tzdata==2024.1 ; python_version >= "3.11" and python_version < "4.0"
urllib3==1.26.19 ; python_version >= "3.11" and python_version < "4.0"
validators==0.29.0 ; python_version >= "3.11" and python_version < "4.0"
wrapt==1.16.0 ; python_version >= "3.11" and python_version < "4.0"
zipp==3.19.2 ; python_version >= "3.11" and python_version < "4.0"
Loading
Loading