pyproject.toml

[tool.poetry]
name = "ol-data-platform"
version = "0.6.1"
description = "Code and configuration for powering the MIT Open Learning data platform"
authors = ["MIT Open Learning Engineering <ol-data@mit.edu>"]
license = "BSD-3-Clause"
readme = "README.md"
repository = "https://github.com/mitodl/ol-data-platform"
classifiers = [
    'Framework :: Dagster',
    'Operating System :: Linux',
    'Programming Language :: Python',
    'Programming Language :: Python :: 3',
    'Programming Language :: Python :: 3.11',
]
packages = [
    { include = "ol_orchestrate", from = "src/" },
]

[tool.poetry.dependencies]
python = "3.12.*"
PyPika = "^0.48.9"
dagster = "^1.8.5"
dagster-airbyte = "^0.25.0"
dagster-aws = "^0.25.0"
dagster-dbt = "^0.25.0"
dagster-docker = "^0.25.0"
dagster-duckdb = "^0.25.0"
dagster-gcp = "^0.25.0"
dagster-postgres = "^0.25.0"
dagster-shell = "^0.25.0"
dagster-slack = "^0.25.0"
dagster-webserver = "^1.8"
dbt-duckdb = "^1.6.0"
dbt-trino = "^1.6.0"
flatten-dict = "^0.4.2"
fsspec = {extras = ["gcs", "s3"], version = "^2024.2.0"}
httpx = "^0.28.0"
hvac = "^2.1.0"
jsonlines = "^4.0.0"
pyarrow = "^18.0.0"
pydantic = "^2.5.3"
pymysql = "^1.0.0"
polars = "^1.0.0"
universal-pathlib = "^0.2.2"

[tool.poetry.group.dev.dependencies]
mypy = "*"
pre-commit = "^4.0.0"
pytest = "^8.0.0"
ruff = "*"
sqlfluff = "^3.0.0"
typer = {extras = ["all"], version = "^0.15.0"}

[build-system]
requires = ["poetry_core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

[tool.sqlfluff.core]
templater = "jinja"
dialect = "hive"
sql_file_exts = ".sql,.sql.j2,.dml,.ddl"
max_line_length = 120
exclude_rules = "CV10"

[tool.sqlfluff.templater]
unwrap_wrapped_queries = true

[tool.sqlfluff.templater.jinja]
apply_dbt_builtins = true
load_macros_from_path = "src/ol_dbt/macros/"

[tool.sqlfluff.templater.dbt]
project_dir = "src/ol_dbt/"
profiles_dir = "src/ol_dbt/"
profile ="open_learning"
target = "qa"

[tool.sqlfluff.layout.type.comma]
# Use leading commas for cleaner diffs
line_position = "leading"

[tool.mypy]
python_version = "3.12"
plugins = [
  "pydantic.mypy"
]
follow_imports = "normal"
warn_redundant_casts = true
warn_unused_ignores = true
disallow_any_generics = true
check_untyped_defs = true
no_implicit_reexport = true
ignore_missing_imports = true
# for strict mypy: (this is the tricky one :-))
# disallow_untyped_defs = true

[tool.pydantic-mypy]
init_forbid_extra = true
init_typed = true
warn_required_dynamic_aliases = true

[tool.ruff]
target-version = "py312"
line-length = 88
lint.select = [
    "A",  # flake8-builtins
    # "AIR",  # Airflow
    # "ANN",  # flake8-annotations
    "ARG",  # flake8-unused-arguments
    # "ASYNC",  # flake8-async
    "B",  # flake8-bugbear
    "BLE",  # flake8-blind-except
    "C4",  # flake8-comprehensions
    "C90",  # mccabe
    # "COM",  # flake8-commas
    "CPY",  # flake8-copyright
    "D",  # pydocstyle
    # "DJ",  # flake8-django
    "DTZ",  # flake8-datetimez
    "E",  # Pycodestyle Error
    "EM",  # flake8-errmsg
    "ERA",  # eradicate
    "EXE",  # flake8-executable
    "F",  # Pyflakes
    "FA",  # flake8-future-annotations
    "FBT",  # flake8-boolean-trap
    "FIX",  # flake8-fixme
    "FLY",  # flynt
    # "FURB",  # refurb
    "G",  # flake8-logging-format
    "I",  # isort
    "ICN",  # flake8-import-conventions
    "INP",  # flake8-no-pep420
    "INT",  # flake8-gettext
    "ISC",  # flake8-implicit-str-concat
    "N",  # pep8-naming
    # "NPY",  # NumPy-specific rules
    # "PD",  # pandas-vet
    "PERF",  # Perflint
    "PGH",  # pygrep-hooks
    "PIE",  # flake8-pie
    "PL",  # Pylint
    "PT",  # flake8-pytest-style
    "PTH",  # flake8-use-pathlib
    "PYI",  # flake8-pyi
    "Q",  # flake8-quotes
    "RET",  # flake8-return
    "RSE",  # flake8-raise
    "RUF",  # Ruff-specific rules
    "S",  # flake8-bandit
    "SIM",  # flake8-simplify
    "SLF",  # flake8-self
    "SLOT",  # flake8-slots
    "T10",  # flake8-debugger
    "T20",  # flake8-print
    # "TCH",  # flake8-type-checking
    "TD",  # flake8-todos
    "TID",  # flake8-tidy-imports
    "TRY",  # tryceratops
    "UP",  # pyupgrade
    "W",  # Pycodestyle Warning
    "YTT",  # flake8-2020
]
lint.ignore = [
    "B008",
    "B905",
    "D104",
    "D200",
    "D202",
    "D205",
    "D301",
    "D400",
    "N803",
    "N806",
    "N999",
    "PIE804",
    "RET505",
    "RET506",
    "RET507",
    "RET508",
    "SIM115",
    "UP007"
]
lint.typing-modules = ["colour.hints"]

[tool.ruff.lint.pydocstyle]
convention = "pep257"

[tool.ruff.lint.flake8-quotes]
inline-quotes = "double"

[tool.ruff.lint.per-file-ignores]
"tests/**" = ["S101"]
"test_*.py" = ["S101"]