Support Python 3.12 and apply ruff as formatter

.github/workflows/format.yml

@@ -13,16 +13,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.12"
     - name: Install dependencies
       run: |
+        python -m pip install --upgrade pip
         python -m pip install poetry
         poetry install
-    - name: run formatter
+    - name: Run formatter
       run: |
-        poetry run yapf --diff --recursive --verbose thunderbolt test
-        poetry run isort --diff thunderbolt test
+        poetry run tox -e ruff

.github/workflows/python_publish.yml

@@ -10,9 +10,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies

.github/workflows/python_test.yml

@@ -14,17 +14,19 @@ jobs:
       max-parallel: 4
       matrix:
         platform: [ubuntu-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install tox tox-gh-actions poetry
+        python -m pip install poetry
+        poetry run python -m pip install tox-gh-actions
+        poetry install
     - name: Test with tox
-      run: tox
+      run: poetry run tox

examples/task.py

@@ -1,8 +1,9 @@
 # define tasks
+from logging import getLogger
+
 import gokart
 import luigi
 from luigi.util import requires
-from logging import getLogger
 
 logger = getLogger(__name__)
 
@@ -25,4 +26,5 @@ def run(self):
         sample = self.load()
         self.dump(sample + f'add task: {self.param}')
 
+
 gokart.run()

pyproject.toml

@@ -7,32 +7,47 @@ license = "MIT"
 readme = "README.md"
 homepage = "https://github.com/m3dev/thunderbolt"
 repository = "https://github.com/m3dev/thunderbolt"
-documentation = ""
 
 [tool.poetry-dynamic-versioning]
 enable = true
 style = "pep440"
 pattern = "^(?P<base>\\d+\\.\\d+\\.\\d+)"
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.12"
+python = ">=3.9,<3.13"
 gokart = "*"
 boto3 = "*"
 tqdm = "*"
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
+ruff = "*"
 tox = "*"
+coverage = "*"
 mock = "*"
-isort = "*"
-yapf = "*"
-toml = "*"
 
-[tool.yapf]
-based_on_style = "pep8"
-column_limit = 160
+[tool.ruff]
+line-length = 160
+exclude = ["venv/*", "tox/*"]
 
-[tool.isort]
-line_length = 160
+[tool.ruff.lint]
+# All the rules are listed on https://docs.astral.sh/ruff/rules/
+extend-select = [
+  "B", # bugbear
+  "I" # isort
+]
+
+# B006: Do not use mutable data structures for argument defaults. They are created during function definition time. All calls to the function reuse this one instance of that data structure, persisting changes between them.
+# B008 Do not perform function calls in argument defaults.  The call is performed only once at function definition time. All calls to your function will reuse the result of that definition-time function call.  If this is intended, assign the function call to a module-level variable and use that variable as a default value.
+ignore = ["B006", "B008"]
+
+# Avoid automatically removing unused imports in __init__.py files.
+ignore-init-module-imports = true
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]
+
+[tool.ruff.format]
+quote-style = "single"
 
 [build-system]
 requires = ["poetry"]

test/sample.py

@@ -3,7 +3,6 @@
 
 import gokart
 import luigi
-from luigi.util import requires
 
 logger = getLogger(__name__)
 

test/test_gcs_client.py

@@ -4,7 +4,6 @@
 
 
 class TestGCSClient(unittest.TestCase):
-
     def setUp(self):
         self.base_path = 'gs://bucket/prefix/'
         self.client = GCSClient(self.base_path, None, None, use_cache=False)

test/test_local_cache.py

@@ -6,7 +6,6 @@
 
 
 class TestLocalCache(unittest.TestCase):
-
     def setUp(self):
         self.base_path = './resources'
         self.local_cache = LocalCache(self.base_path, True)

test/test_local_cache_regression.py

@@ -1,19 +1,15 @@
-import pickle
 import unittest
 from os import path
 
-import pandas as pd
-
 from thunderbolt import Thunderbolt
-from thunderbolt.client.local_cache import LocalCache
+
 """
 requires:
 python sample.py test.TestCaseTask --param=sample --number=1 --workspace-directory=./test_case --local-scheduler
 """
 
 
 class LocalCacheTest(unittest.TestCase):
-
     def test_running(self):
         target = Thunderbolt(self._get_test_case_path(), use_cache=False)
         _ = Thunderbolt(self._get_test_case_path())

test/test_local_directory_client.py

@@ -5,7 +5,6 @@
 
 
 class TestLocalDirectoryClient(unittest.TestCase):
-
     def setUp(self):
         self.client = LocalDirectoryClient('.', None, None, use_cache=False)
 

test/test_s3_client.py

@@ -4,7 +4,6 @@
 
 
 class TestS3Client(unittest.TestCase):
-
     def setUp(self):
         self.base_path = 's3://bucket/prefix/'
         self.client = S3Client(self.base_path, None, None, use_cache=False)

test/test_thunderbolt.py

@@ -12,9 +12,7 @@
 
 
 class TestThunderbolt(unittest.TestCase):
-
     def setUp(self):
-
         def get_tasks():
             return []
 
@@ -35,35 +33,14 @@ def test_get_client(self):
             self.assertEqual(type(output), t)
 
     def test_get_tasks_dic(self):
-        tasks_list = [{
-            'task_name': 'task',
-            'last_modified': 'last_modified_2',
-            'task_params': 'task_params_1',
-            'task_hash': 'task_hash_1',
-            'task_log': 'task_log_1'
-        }, {
-            'task_name': 'task',
-            'last_modified': 'last_modified_1',
-            'task_params': 'task_params_1',
-            'task_hash': 'task_hash_1',
-            'task_log': 'task_log_1'
-        }]
+        tasks_list = [
+            {'task_name': 'task', 'last_modified': 'last_modified_2', 'task_params': 'task_params_1', 'task_hash': 'task_hash_1', 'task_log': 'task_log_1'},
+            {'task_name': 'task', 'last_modified': 'last_modified_1', 'task_params': 'task_params_1', 'task_hash': 'task_hash_1', 'task_log': 'task_log_1'},
+        ]
 
         target = {
-            0: {
-                'task_name': 'task',
-                'last_modified': 'last_modified_1',
-                'task_params': 'task_params_1',
-                'task_hash': 'task_hash_1',
-                'task_log': 'task_log_1'
-            },
-            1: {
-                'task_name': 'task',
-                'last_modified': 'last_modified_2',
-                'task_params': 'task_params_1',
-                'task_hash': 'task_hash_1',
-                'task_log': 'task_log_1'
-            }
+            0: {'task_name': 'task', 'last_modified': 'last_modified_1', 'task_params': 'task_params_1', 'task_hash': 'task_hash_1', 'task_log': 'task_log_1'},
+            1: {'task_name': 'task', 'last_modified': 'last_modified_2', 'task_params': 'task_params_1', 'task_hash': 'task_hash_1', 'task_log': 'task_log_1'},
         }
         output = self.tb._get_tasks_dic(tasks_list=tasks_list)
         self.assertDictEqual(output, target)
@@ -85,27 +62,31 @@ def test_get_task_df(self):
                 'last_modified': 'last_modified_1',
                 'task_params': 'task_params_1',
                 'task_hash': 'task_hash_1',
-                'task_log': 'task_log_1'
+                'task_log': 'task_log_1',
             }
         }
 
-        target = pd.DataFrame({
-            'task_id': ['Task1'],
-            'task_name': ['task_name_1'],
-            'last_modified': ['last_modified_1'],
-            'task_params': ['task_params_1'],
-            'task_hash': ['task_hash_1'],
-            'task_log': ['task_log_1']
-        })
+        target = pd.DataFrame(
+            {
+                'task_id': ['Task1'],
+                'task_name': ['task_name_1'],
+                'last_modified': ['last_modified_1'],
+                'task_params': ['task_params_1'],
+                'task_hash': ['task_hash_1'],
+                'task_log': ['task_log_1'],
+            }
+        )
         output = self.tb.get_task_df(all_data=True)
         pd.testing.assert_frame_equal(output, target)
 
-        target = pd.DataFrame({
-            'task_id': ['Task1'],
-            'task_name': ['task_name_1'],
-            'last_modified': ['last_modified_1'],
-            'task_params': ['task_params_1'],
-        })
+        target = pd.DataFrame(
+            {
+                'task_id': ['Task1'],
+                'task_name': ['task_name_1'],
+                'last_modified': ['last_modified_1'],
+                'task_params': ['task_params_1'],
+            }
+        )
         output = self.tb.get_task_df(all_data=False)
         pd.testing.assert_frame_equal(output, target)
 
@@ -116,15 +97,15 @@ def test_get_data(self):
                 'last_modified': 'last_modified_1',
                 'task_params': 'task_params_1',
                 'task_hash': 'task_hash_1',
-                'task_log': 'task_log_1'
+                'task_log': 'task_log_1',
             },
             'Task2': {
                 'task_name': 'task',
                 'last_modified': 'last_modified_2',
                 'task_params': 'task_params_2',
                 'task_hash': 'task_hash_2',
-                'task_log': 'task_log_2'
-            }
+                'task_log': 'task_log_2',
+            },
         }
         target = 'Task2'
 
@@ -139,22 +120,22 @@ def test_get_data_top_k(self):
                 'last_modified': 'last_modified_1',
                 'task_params': 'task_params_1',
                 'task_hash': 'task_hash_1',
-                'task_log': 'task_log_1'
+                'task_log': 'task_log_1',
             },
             'Task2': {
                 'task_name': 'task',
                 'last_modified': 'last_modified_2',
                 'task_params': 'task_params_2',
                 'task_hash': 'task_hash_2',
-                'task_log': 'task_log_2'
+                'task_log': 'task_log_2',
             },
             'Task3': {
                 'task_name': 'task',
                 'last_modified': 'last_modified_3',
                 'task_params': 'task_params_3',
                 'task_hash': 'task_hash_3',
-                'task_log': 'task_log_3'
-            }
+                'task_log': 'task_log_3',
+            },
         }
         target = ['Task3', 'Task2']
 
@@ -164,16 +145,8 @@ def test_get_data_top_k(self):
 
     def test_load(self):
         self.tb.tasks = {
-            'Task1': {
-                'task_log': {
-                    'file_path': ['./hoge', './piyo']
-                }
-            },
-            'Task2': {
-                'task_log': {
-                    'file_path': ['./hoge']
-                }
-            },
+            'Task1': {'task_log': {'file_path': ['./hoge', './piyo']}},
+            'Task2': {'task_log': {'file_path': ['./hoge']}},
         }
 
         source = 'Task1'
@@ -193,9 +166,7 @@ def test_target_load(self):
         target = 'hoge'
 
         def make_target(file_path):
-
             class mock:
-
                 def __init__(self, file_path):
                     self.file_path = file_path