Update automated tests, pass as-is (#301)

* Update automated tests, pass as-is

* Address PR comments

* Split formatters shell script

* Update and fix tests to run from anywhere

* Update tests

* Not installing pyright on CI

* Not a TODO

* Set pyright's typeCheckingMode to standard

* Use Python scripts instead and check pyright against pylance-prerelease

* Forgot to delete run_formatters

* typing_extensions goes under "Typed libraries and stubs"

* Bump minimum Python version to 3.9

* Missed mypy target version

* Unpin typing_extensions

* post-merge fix

* use docopt-ng instead of docopt

* Add type-var to mypy disable

.github/workflows/test.yml

@@ -29,10 +29,13 @@ jobs:
           python -m pip install -r tests/requirements.txt
 
       - name: Run pyright tests
-        uses: gramster/pyright-action@main
+        uses: jakebailey/pyright-action@v2
         with:
-          project: tests/pyrighttestconfig.json
-          warn-partial: true
+          pylance-version: latest-prerelease
+
+      - name: Run mypy tests
+        run:
+          python -m mypy .
 
   hygiene:
     runs-on: ubuntu-latest

.vscode/settings.json

@@ -2,6 +2,5 @@
   "editor.formatOnSave": false,
   "python.formatting.provider": "black",
   "python.analysis.diagnosticMode": "workspace",
-  "python.analysis.typeCheckingMode": "strict",
   "python.analysis.openFilesOnly": false
 }

pyproject.toml

@@ -1,9 +1,12 @@
 [tool.black]
 line_length = 130
-target_version = ["py37"]
+# Target oldest supported Python version
+target_version = ["py39"]
 
 [tool.ruff]
 line-length = 130
+# Target oldest supported Python version
+target-version = "py39"
 
 [tool.ruff.lint]
 select = [
@@ -61,19 +64,46 @@ extra-standard-library = [
     "zoneinfo",
 ]
 
+[tool.pyright]
+# Target oldest supported Python version
+pythonversion = "3.9"
+typeCheckingMode = "standard"
+# Partial stubs are acceptable
+reportUnknownArgumentType = false
+# Stubs-only packages are fine for testing
+reportMissingModuleSource = false
+# Stubs are allowed to use private variables
+reportPrivateUsage = false
+reportPrivateImportUsage = false
+# Incompatible overrides and property type mismatches are out of our stubs' control
+# as they are inherited from the implementation.
+reportIncompatibleMethodOverride = false
+reportIncompatibleVariableOverride = false
+reportPropertyTypeMismatch = false
+# Overlapping overloads are often necessary in a stub, meaning pyright's check
+# (which is stricter than mypy's; see mypy issue #10143 and #10157)
+# would cause many false positives and catch few bugs.
+reportOverlappingOverload = false
+# The name of the self/cls parameter is out of third-party stubs' control.
+reportSelfClsParameterName = false
+
+# Error reports to fix in code
+reportArgumentType = "none" # TODO
+reportAssertTypeFailure = "none" # TODO
+reportAssignmentType = "none" # TODO
+reportAttributeAccessIssue = "none" # TODO
+reportFunctionMemberAccess = "none" # TODO
+reportGeneralTypeIssues = "none" # TODO
+reportInvalidTypeArguments = "none" # TODO
+reportInvalidTypeForm = "none" # TODO
+reportMissingImports = "none" # TODO
+reportUndefinedVariable = "none" # TODO
+reportUnusedVariable = "none" # TODO
+
 [tool.mypy]
-# Import discovery
-mypy_path = "partial"
-namespace_packages = false
-explicit_package_bases = false
-ignore_missing_imports = true
-follow_imports = "normal"
-follow_imports_for_stubs = false
-no_site_packages = false
-no_silence_site_packages = false
-# Platform configuration
-python_version = "3.8"
-# Disallow dynamic typing
+# Target oldest supported Python version
+python_version = "3.9"
+# Allow dynamic typing
 disallow_any_unimported = false # TODO
 disallow_any_expr = false # TODO
 disallow_any_decorated = false # TODO
@@ -86,25 +116,28 @@ disallow_untyped_defs = false # TODO
 disallow_incomplete_defs = false # TODO
 check_untyped_defs = true
 disallow_untyped_decorators = true
-# None and Optional handling
-no_implicit_optional = true
-strict_optional = true
 # Configuring warnings
 warn_redundant_casts = true
 warn_unused_ignores = false # Change from pandas
-warn_no_return = true
-warn_return_any = false # TODO
-warn_unreachable = false # GH#27396
 # Suppressing errors
-show_none_errors = true
-ignore_errors = false
-# Miscellaneous strictness flags
-allow_untyped_globals = false
-allow_redefinition = false
-local_partial_types = false
-implicit_reexport = true
-strict_equality = true
+disable_error_code = [
+    # Not all imports in these stubs are gonna be typed
+    "import-untyped",
+    # TODO
+    "arg-type",
+    "assignment",
+    "attr-defined",
+    "import-not-found",
+    "misc",
+    "name-defined",
+    "no-redef",
+    "operator",
+    "override",
+    "return",
+    "truthy-function",
+    "type-var",
+    "valid-type",
+    "var-annotated",
+]
 # Configuring error messages
-show_error_context = false
-show_column_numbers = false
-show_error_codes = true
+show_column_numbers = true

tests/matplotlib/figure_tests.py

@@ -1,4 +1,4 @@
-from typing import assert_type
+from typing_extensions import assert_type
 
 from matplotlib.axes import Axes
 from matplotlib.figure import Figure

tests/matplotlib/pyplot_tests.py

@@ -1,4 +1,4 @@
-from typing import Any, assert_type
+from typing_extensions import assert_type
 
 import matplotlib.pyplot as plt
 import numpy as np

tests/requirements.txt

@@ -1,6 +1,22 @@
-matplotlib
-mypy==0.950
+# Tools used for testing
+docopt-ng
+mypy==1.13.*
 pyright
+
+# Typed libraries and stubs
+matplotlib>=3.8
 pytest
+scipy-stubs
+typing_extensions
+
+# Untyped libraries, used to prevent "reportMissingImports" and get inferred typing
+joblib
+networkx
+PyOpenGL
+scikit-image
 scikit-learn
-typing_extensions==4.2.0
+transformers
+vispy
+numpydoc
+pyamg
+traitlets

tests/run_formatters.py

@@ -0,0 +1,39 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def install_requirements():
+    print("\nInstalling requirements...")
+    return subprocess.run((sys.executable, "-m", "pip", "install", "--upgrade", "isort", "black"))
+
+
+def run_isort():
+    print("\nRunning isort...")
+    return subprocess.run((sys.executable, "-m", "isort", "."))
+
+
+def run_black():
+    print("\nRunning Black...")
+    return subprocess.run((sys.executable, "-m", "black", "."))
+
+
+def main():
+    test_folder = Path(__file__).parent
+    root = test_folder.parent
+    os.chdir(root)
+
+    install_requirements().check_returncode()
+    results = (
+        run_isort(),
+        run_black(),
+    )
+    if sum([result.returncode for result in results]) > 0:
+        print("\nOne or more tests failed. See above for details.")
+    else:
+        print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    main()

tests/run_tests.py

@@ -0,0 +1,44 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def install_requirements(test_folder: str):
+    print("\nInstalling requirements...")
+    return subprocess.run(
+        (sys.executable, "-m", "pip", "install", "--upgrade", "-r", os.path.join(test_folder, "requirements.txt"))
+    )
+
+
+def run_pyright():
+    print("\nRunning Pyright...")
+    # https://github.com/RobertCraigie/pyright-python#keeping-pyright-and-pylance-in-sync
+    del os.environ["PYRIGHT_PYTHON_FORCE_VERSION"]
+    os.environ["PYRIGHT_PYTHON_PYLANCE_VERSION"] = "latest-prerelease"
+    return subprocess.run((sys.executable, "-m", "pyright"))
+
+
+def run_mypy():
+    print("\nRunning mypy...")
+    return subprocess.run((sys.executable, "-m", "mypy"))
+
+
+def main():
+    test_folder = Path(__file__).parent
+    root = test_folder.parent
+    os.chdir(root)
+
+    install_requirements(test_folder).check_returncode()
+    results = (
+        run_mypy(),
+        run_pyright(),
+    )
+    if sum([result.returncode for result in results]) > 0:
+        print("\nOne or more tests failed. See above for details.")
+    else:
+        print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    main()